feat: 训练数据管理

package.json

@@ -27,6 +27,7 @@
     "hyperdown": "^2.4.29",
     "immer": "^9.0.19",
     "jsonwebtoken": "^9.0.0",
+    "mammoth": "^1.5.1",
     "mongoose": "^6.10.0",
     "nanoid": "^4.0.1",
     "next": "13.1.6",

pnpm-lock.yaml

@@ -31,6 +31,7 @@ specifiers:
   immer: ^9.0.19
   jsonwebtoken: ^9.0.0
   lint-staged: ^13.1.2
+  mammoth: ^1.5.1
   mongoose: ^6.10.0
   nanoid: ^4.0.1
   next: 13.1.6
@@ -69,6 +70,7 @@ dependencies:
   hyperdown: registry.npmmirror.com/hyperdown/2.4.29
   immer: registry.npmmirror.com/immer/9.0.19
   jsonwebtoken: registry.npmmirror.com/jsonwebtoken/9.0.0
+  mammoth: registry.npmmirror.com/mammoth/1.5.1
   mongoose: registry.npmmirror.com/mongoose/6.10.0
   nanoid: registry.npmmirror.com/nanoid/4.0.1
   next: registry.npmmirror.com/next/13.1.6_wiv434v7erz4aedd5whhdwmpv4
@@ -5085,6 +5087,14 @@ packages:
       picomatch: registry.npmmirror.com/picomatch/2.3.1
     dev: false
 
+  registry.npmmirror.com/argparse/1.0.10:
+    resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/argparse/-/argparse-1.0.10.tgz}
+    name: argparse
+    version: 1.0.10
+    dependencies:
+      sprintf-js: registry.npmmirror.com/sprintf-js/1.0.3
+    dev: false
+
   registry.npmmirror.com/argparse/2.0.1:
     resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/argparse/-/argparse-2.0.1.tgz}
     name: argparse
@@ -5333,6 +5343,12 @@ packages:
       readable-stream: registry.npmmirror.com/readable-stream/3.6.1
     dev: false
 
+  registry.npmmirror.com/bluebird/3.4.7:
+    resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/bluebird/-/bluebird-3.4.7.tgz}
+    name: bluebird
+    version: 3.4.7
+    dev: false
+
   registry.npmmirror.com/boolbase/1.0.0:
     resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz}
     name: boolbase
@@ -5672,6 +5688,12 @@ packages:
       browserslist: registry.npmmirror.com/browserslist/4.21.5
     dev: true
 
+  registry.npmmirror.com/core-util-is/1.0.3:
+    resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz}
+    name: core-util-is
+    version: 1.0.3
+    dev: false
+
   registry.npmmirror.com/cosmiconfig/7.1.0:
     resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz}
     name: cosmiconfig
@@ -5912,6 +5934,12 @@ packages:
     engines: {node: '>=0.3.1'}
     dev: false
 
+  registry.npmmirror.com/dingbat-to-unicode/1.0.1:
+    resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz}
+    name: dingbat-to-unicode
+    version: 1.0.1
+    dev: false
+
   registry.npmmirror.com/dir-glob/3.0.1:
     resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dir-glob/-/dir-glob-3.0.1.tgz}
     name: dir-glob
@@ -5974,6 +6002,14 @@ packages:
       domhandler: registry.npmmirror.com/domhandler/4.3.1
     dev: true
 
+  registry.npmmirror.com/duck/0.1.12:
+    resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/duck/-/duck-0.1.12.tgz}
+    name: duck
+    version: 0.1.12
+    dependencies:
+      underscore: registry.npmmirror.com/underscore/1.13.6
+    dev: false
+
   registry.npmmirror.com/eastasianwidth/0.2.0:
     resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz}
     name: eastasianwidth
@@ -7133,6 +7169,12 @@ packages:
     engines: {node: '>= 4'}
     dev: true
 
+  registry.npmmirror.com/immediate/3.0.6:
+    resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/immediate/-/immediate-3.0.6.tgz}
+    name: immediate
+    version: 3.0.6
+    dev: false
+
   registry.npmmirror.com/immer/9.0.19:
     resolution: {integrity: sha512-eY+Y0qcsB4TZKwgQzLaE/lqYMlKhv5J9dyd2RhhtGhNo2njPXDqU9XPfcNfa3MIDsdtZt5KlkIsirlo4dHsWdQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/immer/-/immer-9.0.19.tgz}
     name: immer
@@ -7506,6 +7548,12 @@ packages:
       is-docker: registry.npmmirror.com/is-docker/2.2.1
     dev: true
 
+  registry.npmmirror.com/isarray/1.0.0:
+    resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/isarray/-/isarray-1.0.0.tgz}
+    name: isarray
+    version: 1.0.0
+    dev: false
+
   registry.npmmirror.com/isarray/2.0.5:
     resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/isarray/-/isarray-2.0.5.tgz}
     name: isarray
@@ -7609,6 +7657,17 @@ packages:
       object.assign: registry.npmmirror.com/object.assign/4.1.4
     dev: true
 
+  registry.npmmirror.com/jszip/3.10.1:
+    resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jszip/-/jszip-3.10.1.tgz}
+    name: jszip
+    version: 3.10.1
+    dependencies:
+      lie: registry.npmmirror.com/lie/3.3.0
+      pako: registry.npmmirror.com/pako/1.0.11
+      readable-stream: registry.npmmirror.com/readable-stream/2.3.8
+      setimmediate: registry.npmmirror.com/setimmediate/1.0.5
+    dev: false
+
   registry.npmmirror.com/jwa/1.4.1:
     resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jwa/-/jwa-1.4.1.tgz}
     name: jwa
@@ -7684,6 +7743,14 @@ packages:
       type-check: registry.npmmirror.com/type-check/0.4.0
     dev: true
 
+  registry.npmmirror.com/lie/3.3.0:
+    resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lie/-/lie-3.3.0.tgz}
+    name: lie
+    version: 3.3.0
+    dependencies:
+      immediate: registry.npmmirror.com/immediate/3.0.6
+    dev: false
+
   registry.npmmirror.com/lilconfig/2.0.6:
     resolution: {integrity: sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lilconfig/-/lilconfig-2.0.6.tgz}
     name: lilconfig
@@ -7801,6 +7868,16 @@ packages:
     dependencies:
       js-tokens: registry.npmmirror.com/js-tokens/4.0.0
 
+  registry.npmmirror.com/lop/0.4.1:
+    resolution: {integrity: sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lop/-/lop-0.4.1.tgz}
+    name: lop
+    version: 0.4.1
+    dependencies:
+      duck: registry.npmmirror.com/duck/0.1.12
+      option: registry.npmmirror.com/option/0.2.4
+      underscore: registry.npmmirror.com/underscore/1.13.6
+    dev: false
+
   registry.npmmirror.com/lowlight/1.20.0:
     resolution: {integrity: sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lowlight/-/lowlight-1.20.0.tgz}
     name: lowlight
@@ -7826,6 +7903,23 @@ packages:
     dependencies:
       yallist: registry.npmmirror.com/yallist/4.0.0
 
+  registry.npmmirror.com/mammoth/1.5.1:
+    resolution: {integrity: sha512-7ZioZBf/1HjYrm1qZJOO+DD+rYxLvwrHS+HVOwW89hwIp+r6ZqJ/Eq2rXSS+8ezZ3/DuW6FUUp2Dfz6e7B2pBQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mammoth/-/mammoth-1.5.1.tgz}
+    name: mammoth
+    version: 1.5.1
+    hasBin: true
+    dependencies:
+      argparse: registry.npmmirror.com/argparse/1.0.10
+      bluebird: registry.npmmirror.com/bluebird/3.4.7
+      dingbat-to-unicode: registry.npmmirror.com/dingbat-to-unicode/1.0.1
+      jszip: registry.npmmirror.com/jszip/3.10.1
+      lop: registry.npmmirror.com/lop/0.4.1
+      path-is-absolute: registry.npmmirror.com/path-is-absolute/1.0.1
+      sax: registry.npmmirror.com/sax/1.1.6
+      underscore: registry.npmmirror.com/underscore/1.13.6
+      xmlbuilder: registry.npmmirror.com/xmlbuilder/10.1.1
+    dev: false
+
   registry.npmmirror.com/markdown-table/3.0.3:
     resolution: {integrity: sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/markdown-table/-/markdown-table-3.0.3.tgz}
     name: markdown-table
@@ -8750,6 +8844,12 @@ packages:
       - debug
     dev: false
 
+  registry.npmmirror.com/option/0.2.4:
+    resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/option/-/option-0.2.4.tgz}
+    name: option
+    version: 0.2.4
+    dev: false
+
   registry.npmmirror.com/optionator/0.9.1:
     resolution: {integrity: sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/optionator/-/optionator-0.9.1.tgz}
     name: optionator
@@ -8791,6 +8891,12 @@ packages:
       aggregate-error: registry.npmmirror.com/aggregate-error/3.1.0
     dev: true
 
+  registry.npmmirror.com/pako/1.0.11:
+    resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pako/-/pako-1.0.11.tgz}
+    name: pako
+    version: 1.0.11
+    dev: false
+
   registry.npmmirror.com/parent-module/1.0.1:
     resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parent-module/-/parent-module-1.0.1.tgz}
     name: parent-module
@@ -8841,7 +8947,6 @@ packages:
     name: path-is-absolute
     version: 1.0.1
     engines: {node: '>=0.10.0'}
-    dev: true
 
   registry.npmmirror.com/path-key/3.1.1:
     resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz}
@@ -8948,6 +9053,12 @@ packages:
     engines: {node: '>=6'}
     dev: false
 
+  registry.npmmirror.com/process-nextick-args/2.0.1:
+    resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz}
+    name: process-nextick-args
+    version: 2.0.1
+    dev: false
+
   registry.npmmirror.com/prop-types/15.8.1:
     resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/prop-types/-/prop-types-15.8.1.tgz}
     name: prop-types
@@ -9211,6 +9322,20 @@ packages:
       loose-envify: registry.npmmirror.com/loose-envify/1.4.0
     dev: false
 
+  registry.npmmirror.com/readable-stream/2.3.8:
+    resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/readable-stream/-/readable-stream-2.3.8.tgz}
+    name: readable-stream
+    version: 2.3.8
+    dependencies:
+      core-util-is: registry.npmmirror.com/core-util-is/1.0.3
+      inherits: registry.npmmirror.com/inherits/2.0.4
+      isarray: registry.npmmirror.com/isarray/1.0.0
+      process-nextick-args: registry.npmmirror.com/process-nextick-args/2.0.1
+      safe-buffer: registry.npmmirror.com/safe-buffer/5.1.2
+      string_decoder: registry.npmmirror.com/string_decoder/1.1.1
+      util-deprecate: registry.npmmirror.com/util-deprecate/1.0.2
+    dev: false
+
   registry.npmmirror.com/readable-stream/3.6.1:
     resolution: {integrity: sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.1.tgz}
     name: readable-stream
@@ -9467,6 +9592,12 @@ packages:
       mri: registry.npmmirror.com/mri/1.2.0
     dev: false
 
+  registry.npmmirror.com/safe-buffer/5.1.2:
+    resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz}
+    name: safe-buffer
+    version: 5.1.2
+    dev: false
+
   registry.npmmirror.com/safe-buffer/5.2.1:
     resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz}
     name: safe-buffer
@@ -9506,6 +9637,12 @@ packages:
       source-map-js: registry.npmmirror.com/source-map-js/1.0.2
     dev: false
 
+  registry.npmmirror.com/sax/1.1.6:
+    resolution: {integrity: sha512-8zci48uUQyfqynGDSkUMD7FCJB96hwLnlZOXlgs1l3TX+LW27t3psSWKUxC0fxVgA86i8tL4NwGcY1h/6t3ESg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sax/-/sax-1.1.6.tgz}
+    name: sax
+    version: 1.1.6
+    dev: false
+
   registry.npmmirror.com/scheduler/0.23.0:
     resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz}
     name: scheduler
@@ -9530,6 +9667,12 @@ packages:
     dependencies:
       lru-cache: registry.npmmirror.com/lru-cache/6.0.0
 
+  registry.npmmirror.com/setimmediate/1.0.5:
+    resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/setimmediate/-/setimmediate-1.0.5.tgz}
+    name: setimmediate
+    version: 1.0.5
+    dev: false
+
   registry.npmmirror.com/sharp/0.31.3:
     resolution: {integrity: sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sharp/-/sharp-0.31.3.tgz}
     name: sharp
@@ -9713,6 +9856,12 @@ packages:
     dev: false
     optional: true
 
+  registry.npmmirror.com/sprintf-js/1.0.3:
+    resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sprintf-js/-/sprintf-js-1.0.3.tgz}
+    name: sprintf-js
+    version: 1.0.3
+    dev: false
+
   registry.npmmirror.com/stable/0.1.8:
     resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/stable/-/stable-0.1.8.tgz}
     name: stable
@@ -9793,6 +9942,14 @@ packages:
       es-abstract: registry.npmmirror.com/es-abstract/1.21.1
     dev: true
 
+  registry.npmmirror.com/string_decoder/1.1.1:
+    resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/string_decoder/-/string_decoder-1.1.1.tgz}
+    name: string_decoder
+    version: 1.1.1
+    dependencies:
+      safe-buffer: registry.npmmirror.com/safe-buffer/5.1.2
+    dev: false
+
   registry.npmmirror.com/string_decoder/1.3.0:
     resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/string_decoder/-/string_decoder-1.3.0.tgz}
     name: string_decoder
@@ -10149,6 +10306,12 @@ packages:
       which-boxed-primitive: registry.npmmirror.com/which-boxed-primitive/1.0.2
     dev: true
 
+  registry.npmmirror.com/underscore/1.13.6:
+    resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/underscore/-/underscore-1.13.6.tgz}
+    name: underscore
+    version: 1.13.6
+    dev: false
+
   registry.npmmirror.com/unicode-canonical-property-names-ecmascript/2.0.0:
     resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz}
     name: unicode-canonical-property-names-ecmascript
@@ -10489,6 +10652,13 @@ packages:
     name: wrappy
     version: 1.0.2
 
+  registry.npmmirror.com/xmlbuilder/10.1.1:
+    resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
+    name: xmlbuilder
+    version: 10.1.1
+    engines: {node: '>=4.0'}
+    dev: false
+
   registry.npmmirror.com/xtend/4.0.2:
     resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
     name: xtend

src/api/data.ts

@@ -0,0 +1,13 @@
+import { GET, POST, DELETE, PUT } from './request';
+import { RequestPaging } from '../types/index';
+import { Obj2Query } from '@/utils/tools';
+import type { DataListItem } from '@/types/data';
+import type { PagingData } from '../types/index';
+
+export const getDataList = (data: RequestPaging) =>
+  GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
+
+export const postData = (name: string) => POST<string>(`/data/postData?name=${name}`);
+
+export const postSplitData = (dataId: string, text: string) =>
+  POST(`/data/splitData`, { dataId, text });

src/components/Layout/index.tsx

@@ -26,12 +26,12 @@ const navbarList = [
     link: '/model/list',
     activeLink: ['/model/list', '/model/detail']
   },
-  // {
+  {
-  //   label: '数据',
+    label: '数据',
-  //   icon: 'icon-datafull',
+    icon: 'icon-datafull',
-  //   link: '/training/dataList',
+    link: '/data/list',
-  //   activeLink: ['/training/dataList']
+    activeLink: ['/data/list']
-  // },
+  },
   {
     label: '账号',
     icon: 'icon-yonghu-yuan',
@@ -62,8 +62,8 @@ const Layout = ({ children }: { children: JSX.Element }) => {
               <Box h={'100%'} position={'fixed'} left={0} top={0} w={'80px'}>
                 <Navbar navbarList={navbarList} />
               </Box>
-              <Box ml={'80px'} p={7}>
+              <Box ml={'80px'} p={7} h={'100%'}>
-                <Box maxW={'1100px'} m={'auto'}>
+                <Box maxW={'1100px'} m={'auto'} h={'100%'}>
                   <Auth>{children}</Auth>
                 </Box>
               </Box>

src/components/ScrollData/index.tsx

@@ -0,0 +1,18 @@
+import React from 'react';
+import type { BoxProps } from '@chakra-ui/react';
+import { Box } from '@chakra-ui/react';
+
+interface Props extends BoxProps {
+  nextPage: () => void;
+  children: React.ReactNode;
+}
+
+const ScrollData = ({ children, nextPage, ...props }: Props) => {
+  return (
+    <Box {...props} overflow={'auto'}>
+      {children}
+    </Box>
+  );
+};
+
+export default ScrollData;

src/hooks/useConfirm.tsx

@@ -19,9 +19,10 @@ export const useConfirm = ({ title = '提示', content }: { title?: string; cont
   return {
     openConfirm: useCallback(
       (confirm?: any, cancel?: any) => {
-        onOpen();
         confirmCb.current = confirm;
         cancelCb.current = cancel;
+
+        return onOpen;
       },
       [onOpen]
     ),

src/hooks/usePaging.ts

@@ -1,16 +1,18 @@
 import { useState, useCallback } from 'react';
 import type { PagingData } from '../types/index';
 import { useQuery } from '@tanstack/react-query';
+import { useToast } from './useToast';
 
 export const usePaging = <T = any>({
   api,
   pageSize = 10,
-  params
+  params = {}
 }: {
   api: (data: any) => Promise<PagingData<T>>;
   pageSize?: number;
   params?: Record<string, any>;
 }) => {
+  const { toast } = useToast();
   const [data, setData] = useState<T[]>([]);
   const [pageNum, setPageNum] = useState(1);
   const [total, setTotal] = useState(0);
@@ -18,36 +20,40 @@ export const usePaging = <T = any>({
   const [requesting, setRequesting] = useState(false);
 
   const getData = useCallback(
-    async (init = false) => {
+    async (num: number, init = false) => {
       if (requesting) return;
       if (!init && isLoadAll) return;
       setRequesting(true);
 
       try {
         const res = await api({
-          pageNum,
+          pageNum: num,
           pageSize,
-          ...(params ? params : {})
+          ...params
         });
         setData((state) => {
           const data = init ? res.data : state.concat(res.data);
           if (data.length >= res.total) {
             setIsLoadAll(true);
           }
+          setTotal(res.total);
           return data;
         });
-        setTotal(res.total);
+      } catch (error: any) {
-      } catch (error) {
+        toast({
+          title: error?.message || '获取数据异常',
+          status: 'error'
+        });
         console.log(error);
       }
 
       setRequesting(false);
       return null;
     },
-    [api, isLoadAll, pageNum, pageSize, params, requesting]
+    [api, isLoadAll, pageSize, params, requesting, toast]
   );
 
-  useQuery(['init', pageNum], () => getData(pageNum === 1));
+  useQuery(['init', pageNum], () => getData(pageNum, pageNum === 1));
 
   return {
     pageNum,
@@ -55,6 +61,7 @@ export const usePaging = <T = any>({
     setPageNum,
     total,
     data,
-    getData
+    getData,
+    requesting
   };
 };

src/hooks/useSelectFile.tsx

@@ -0,0 +1,34 @@
+import React, { useRef, useCallback } from 'react';
+import { Box } from '@chakra-ui/react';
+
+export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
+  const { fileType = '*', multiple = false } = props || {};
+  const SelectFileDom = useRef<HTMLInputElement>(null);
+
+  const File = useCallback(
+    ({ onSelect }: { onSelect: (e: File[]) => void }) => (
+      <Box position={'absolute'} w={0} h={0} overflow={'hidden'}>
+        <input
+          ref={SelectFileDom}
+          type="file"
+          accept={fileType}
+          multiple={multiple}
+          onChange={(e) => {
+            if (!e.target.files || e.target.files?.length === 0) return;
+            onSelect(Array.from(e.target.files));
+          }}
+        />
+      </Box>
+    ),
+    [fileType, multiple]
+  );
+
+  const onOpen = useCallback(() => {
+    SelectFileDom.current && SelectFileDom.current.click();
+  }, []);
+
+  return {
+    File,
+    onOpen
+  };
+};

src/hooks/useTabs.tsx

@@ -0,0 +1,18 @@
+import React, { useState, useCallback, useRef } from 'react';
+
+export const useTabs = ({
+  tabs = []
+}: {
+  tabs: {
+    id: string;
+    label: string;
+  }[];
+}) => {
+  const [activeTab, setActiveTab] = useState(tabs[0].id);
+
+  return {
+    tabs,
+    activeTab,
+    setActiveTab
+  };
+};

src/pages/_app.tsx

@@ -38,8 +38,9 @@ export default function App({ Component, pageProps }: AppProps) {
         />
         <link rel="icon" href="/favicon.ico" />
       </Head>
-      <Script src="/iconfont.js" strategy="afterInteractive"></Script>
+      <Script src="/js/iconfont.js" strategy="afterInteractive"></Script>
-      <Script src="/qrcode.min.js" strategy="afterInteractive"></Script>
+      <Script src="/js/qrcode.min.js" strategy="afterInteractive"></Script>
+      <Script src="/js/pdf.js" strategy="afterInteractive"></Script>
       <QueryClientProvider client={queryClient}>
         <ChakraProvider theme={theme}>
           <ColorModeScript initialColorMode={theme.config.initialColorMode} />

src/pages/api/data/getDataList.ts

@@ -0,0 +1,88 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Data, DataItem } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import type { DataSchema } from '@/types/mongoSchema';
+import type { DataListItem } from '@/types/data';
+import type { PagingData } from '@/types';
+import mongoose from 'mongoose';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { authorization } = req.headers;
+    let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
+
+    pageNum = +pageNum;
+    pageSize = +pageSize;
+
+    if (!authorization) {
+      throw new Error('缺少登录凭证');
+    }
+
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+
+    // 根据 id 获取用户账单
+    const datalist = await Data.aggregate<DataListItem>([
+      {
+        $match: {
+          userId: new mongoose.Types.ObjectId(userId)
+        }
+      },
+      {
+        $sort: { createTime: -1 } // 按照创建时间倒序排列
+      },
+      {
+        $skip: (pageNum - 1) * pageSize // 跳过前面的数据
+      },
+      {
+        $limit: pageSize // 取出指定数量的数据
+      },
+      {
+        $lookup: {
+          from: 'dataitems',
+          localField: '_id',
+          foreignField: 'dataId',
+          as: 'items'
+        }
+      },
+      {
+        $addFields: {
+          totalData: {
+            $size: '$items' // 统计dataItem的总数
+          },
+          trainingData: {
+            $size: {
+              $filter: {
+                input: '$items',
+                as: 'item',
+                cond: { $eq: ['$$item.status', 1] } // 统计status为1的数量
+              }
+            }
+          }
+        }
+      },
+      {
+        $project: {
+          items: 0 // 不返回 items 字段
+        }
+      }
+    ]);
+
+    jsonRes<PagingData<DataListItem>>(res, {
+      data: {
+        pageNum,
+        pageSize,
+        data: datalist,
+        total: 1
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/data/postData.ts

@@ -0,0 +1,33 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Data } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    let { name } = req.query as { name: string };
+    if (!name) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    // 生成 data 集合
+    const data = await Data.create({
+      userId,
+      name
+    });
+
+    jsonRes(res, {
+      data: data._id
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/data/splitData.ts

@@ -4,11 +4,10 @@ import { connectToDatabase, Data, DataItem } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
 import { generateQA } from '@/service/events/generateQA';
 
-/* 定时删除那些不活跃的内容 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    let { text, name } = req.body as { text: string; name: string };
+    let { text, dataId } = req.body as { text: string; dataId: string };
-    if (!text || !name) {
+    if (!text || !dataId) {
       throw new Error('参数错误');
     }
     text = text.replace(/\n+/g, '\n');
@@ -18,28 +17,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
     const userId = await authToken(authorization);
 
-    // 生成 data 父级
-    const data = await Data.create({
-      userId,
-      name
-    });
-
     const dataItems: any[] = [];
 
     // 格式化文本长度
     for (let i = 0; i <= text.length / 1000; i++) {
-      const dataItem = {
+      dataItems.push({
+        temperature: 0,
         userId,
-        dataId: data._id,
+        dataId,
         text: text.slice(i * 1000, (i + 1) * 1000),
         status: 1
-      };
-
-      [0, 0.2, 0.4, 0.6, 0.8, 1.0].forEach((temperature) => {
-        dataItems.push({
-          temperature,
-          ...dataItem
-        });
       });
     }
 
@@ -58,8 +45,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     });
   }
 }
-
-/**
- * 检查文本是否按格式返回
- */
-function splitText(text: string) {}

src/pages/data/components/CreateDataModal.tsx

@@ -0,0 +1,65 @@
+import React, { useState } from 'react';
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalFooter,
+  ModalBody,
+  ModalCloseButton,
+  Button,
+  Input
+} from '@chakra-ui/react';
+import { postData } from '@/api/data';
+import { useMutation } from '@tanstack/react-query';
+
+const CreateDataModal = ({
+  onClose,
+  onSuccess
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+}) => {
+  const [inputVal, setInputVal] = useState('');
+
+  const { isLoading, mutate } = useMutation({
+    mutationFn: (name: string) => postData(name),
+    onSuccess() {
+      onSuccess();
+      onClose();
+    }
+  });
+
+  return (
+    <Modal isOpen={true} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>创建数据集</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody display={'flex'}>
+          <Input
+            value={inputVal}
+            onChange={(e) => setInputVal(e.target.value)}
+            placeholder={'数据集名称'}
+          ></Input>
+        </ModalBody>
+        <ModalFooter>
+          <Button colorScheme={'gray'} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            ml={3}
+            isDisabled={inputVal === ''}
+            isLoading={isLoading}
+            onClick={() => mutate(inputVal)}
+          >
+            确认
+          </Button>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+export default CreateDataModal;

src/pages/data/components/ImportDataModal.tsx

@@ -0,0 +1,175 @@
+import React, { useState, useCallback } from 'react';
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalFooter,
+  ModalBody,
+  ModalCloseButton,
+  Button,
+  Input,
+  Box,
+  Flex,
+  Textarea
+} from '@chakra-ui/react';
+import { useTabs } from '@/hooks/useTabs';
+import { useConfirm } from '@/hooks/useConfirm';
+import { useSelectFile } from '@/hooks/useSelectFile';
+import { readTxtContent, readPdfContent, readDocContent } from '@/utils/tools';
+import { postSplitData } from '@/api/data';
+import { useMutation } from '@tanstack/react-query';
+import { useToast } from '@/hooks/useToast';
+import { useLoading } from '@/hooks/useLoading';
+
+const ImportDataModal = ({ dataId, onClose }: { dataId: string; onClose: () => void }) => {
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认提交生成任务？该任务无法终止！'
+  });
+  const { toast } = useToast();
+  const { setIsLoading, Loading } = useLoading();
+  const { File, onOpen } = useSelectFile({ fileType: '.txt,.doc,.docx,.pdf', multiple: true });
+  const { tabs, activeTab, setActiveTab } = useTabs({
+    tabs: [
+      { id: 'text', label: '文本' },
+      { id: 'doc', label: '文件' },
+      { id: 'url', label: '链接' }
+    ]
+  });
+
+  const [textInput, setTextInput] = useState('');
+  const [fileText, setFileText] = useState('');
+
+  const { mutate: handleClickSubmit, isLoading } = useMutation({
+    mutationFn: async () => {
+      let text = '';
+      if (activeTab === 'text') {
+        text = textInput;
+      } else if (activeTab === 'doc') {
+        text = fileText;
+      } else if (activeTab === 'url') {
+      }
+      if (!text) return;
+      return postSplitData(dataId, text);
+    },
+    onSuccess() {
+      toast({
+        title: '任务提交成功',
+        status: 'success'
+      });
+      onClose();
+    },
+    onError(err: any) {
+      toast({
+        title: err?.message || '提交任务异常',
+        status: 'error'
+      });
+    }
+  });
+
+  const onSelectFile = useCallback(
+    async (e: File[]) => {
+      setIsLoading(true);
+      try {
+        const fileTexts = (
+          await Promise.all(
+            e.map((file) => {
+              // @ts-ignore
+              const extension = file?.name?.split('.').pop().toLowerCase();
+              if (extension === 'txt') {
+                return readTxtContent(file);
+              } else if (extension === 'pdf') {
+                return readPdfContent(file);
+              } else if (extension === 'docx' || extension === 'doc') {
+                return readDocContent(file);
+              }
+              return '';
+            })
+          )
+        ).join('\n');
+        setFileText(fileTexts);
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: typeof error === 'string' ? error : '解析文件失败',
+          status: 'error'
+        });
+      }
+      setIsLoading(false);
+    },
+    [setIsLoading, toast]
+  );
+
+  return (
+    <Modal isOpen={true} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent position={'relative'} maxW={['90vw', '800px']}>
+        <ModalHeader>导入数据，生成QA</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody display={'flex'}>
+          <Box>
+            {tabs.map((item) => (
+              <Button
+                key={item.id}
+                display={'block'}
+                variant={activeTab === item.id ? 'solid' : 'outline'}
+                _notLast={{
+                  mb: 3
+                }}
+                onClick={() => setActiveTab(item.id)}
+              >
+                {item.label}
+              </Button>
+            ))}
+          </Box>
+
+          <Box flex={'1 0 0'} w={0} ml={3} minH={'200px'}>
+            {activeTab === 'text' && (
+              <Textarea
+                h={'100%'}
+                maxLength={-1}
+                value={textInput}
+                placeholder={'请粘贴或输入需要处理的文本'}
+                onChange={(e) => setTextInput(e.target.value)}
+              />
+            )}
+            {activeTab === 'doc' && (
+              <Flex
+                flexDirection={'column'}
+                h={'100%'}
+                alignItems={'center'}
+                justifyContent={'center'}
+                border={'1px solid '}
+                borderColor={'blackAlpha.200'}
+                borderRadius={'md'}
+              >
+                <Button onClick={onOpen}>选择文件</Button>
+                {fileText && <Box mt={2}>一共 {fileText.length} 个字</Box>}
+              </Flex>
+            )}
+          </Box>
+        </ModalBody>
+        <ModalFooter>
+          <Button colorScheme={'gray'} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            ml={3}
+            isLoading={isLoading}
+            isDisabled={!textInput && !fileText}
+            onClick={openConfirm(handleClickSubmit)}
+          >
+            确认
+          </Button>
+        </ModalFooter>
+        <Loading />
+      </ModalContent>
+
+      <ConfirmChild />
+      <File onSelect={onSelectFile} />
+    </Modal>
+  );
+};
+
+export default ImportDataModal;

src/pages/data/list.tsx

@@ -0,0 +1,111 @@
+import React, { useState } from 'react';
+import {
+  Card,
+  Box,
+  Flex,
+  Button,
+  Table,
+  Thead,
+  Tbody,
+  Tr,
+  Th,
+  Td,
+  TableContainer,
+  useDisclosure
+} from '@chakra-ui/react';
+import { getDataList } from '@/api/data';
+import { usePaging } from '@/hooks/usePaging';
+import type { DataListItem } from '@/types/data';
+import ScrollData from '@/components/ScrollData';
+import dayjs from 'dayjs';
+import dynamic from 'next/dynamic';
+
+const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
+const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
+
+const DataList = () => {
+  const {
+    setPageNum,
+    pageNum,
+    data: dataList,
+    getData
+  } = usePaging<DataListItem>({
+    api: getDataList,
+    pageSize: 20
+  });
+  const [ImportDataId, setImportDataId] = useState<string>();
+
+  const {
+    isOpen: isOpenCreateDataModal,
+    onOpen: onOpenCreateDataModal,
+    onClose: onCloseCreateDataModal
+  } = useDisclosure();
+
+  return (
+    <Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
+      <Card px={6} py={4}>
+        <Flex>
+          <Box flex={1} mr={1}>
+            <Box fontSize={'xl'} fontWeight={'bold'}>
+              对话数据管理
+            </Box>
+            <Box fontSize={'xs'} color={'blackAlpha.600'}>
+              允许你将任意文本数据拆分成 QA 的形式。你可以使用这些 QA 去微调你的对话模型。
+            </Box>
+          </Box>
+          <Button variant={'outline'} onClick={onOpenCreateDataModal}>
+            创建数据集
+          </Button>
+        </Flex>
+      </Card>
+      {/* 数据表 */}
+      <Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
+        <ScrollData h={'100%'} nextPage={() => setPageNum(pageNum + 1)}>
+          <TableContainer>
+            <Table>
+              <Thead>
+                <Tr>
+                  <Th>集合名</Th>
+                  <Th>创建时间</Th>
+                  <Th>训练中 / 总数据</Th>
+                  <Th></Th>
+                </Tr>
+              </Thead>
+              <Tbody>
+                {dataList.map((item, i) => (
+                  <Tr key={item._id}>
+                    <Td>{item.name}</Td>
+                    <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
+                    <Td>
+                      {item.trainingData} / {item.totalData}
+                    </Td>
+                    <Td>
+                      <Button
+                        size={'sm'}
+                        variant={'outline'}
+                        mr={2}
+                        onClick={() => setImportDataId(item._id)}
+                      >
+                        导入
+                      </Button>
+                      <Button size={'sm'}>导出</Button>
+                    </Td>
+                  </Tr>
+                ))}
+              </Tbody>
+            </Table>
+          </TableContainer>
+        </ScrollData>
+      </Card>
+
+      {ImportDataId && (
+        <ImportDataModal dataId={ImportDataId} onClose={() => setImportDataId(undefined)} />
+      )}
+      {isOpenCreateDataModal && (
+        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
+      )}
+    </Box>
+  );
+};
+
+export default DataList;

src/pages/model/detail.tsx

@@ -291,15 +291,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
           </Box>
           <Flex mt={5} alignItems={'center'}>
             <Box flex={'0 0 80px'}>删除模型:</Box>
-            <Button
+            <Button colorScheme={'red'} size={'sm'} onClick={openConfirm(handleDelModel)}>
-              colorScheme={'red'}
-              size={'sm'}
-              onClick={() => {
-                openConfirm(() => {
-                  handleDelModel();
-                });
-              }}
-            >
               删除模型
             </Button>
           </Flex>

src/pages/training/dataList.tsx

@@ -1,23 +0,0 @@
-import React from 'react';
-import { Card, Box, Flex, Button } from '@chakra-ui/react';
-
-const TrainDataList = () => {
-  return (
-    <>
-      <Card px={6} py={4}>
-        <Flex alignItems={'center'}>
-          <Box fontSize={'xl'} fontWeight={'bold'} flex={1}>
-            训练数据管理
-          </Box>
-          <Button variant={'outline'} mr={6}>
-            导入数据
-          </Button>
-          <Button>插入一条数据</Button>
-        </Flex>
-      </Card>
-      {/* 数据表 */}
-    </>
-  );
-};
-
-export default TrainDataList;

src/service/events/generateQA.ts

@@ -23,7 +23,7 @@ export async function generateQA(next = false): Promise<any> {
   try {
     // 找出一个需要生成的 dataItem
     dataItem = await DataItem.findOne({
-      status: 1,
+      status: { $ne: 0 },
       times: { $gt: 0 }
     });
 
@@ -56,7 +56,8 @@ export async function generateQA(next = false): Promise<any> {
       throw new Error('获取 openai key 失败');
     }
 
-    console.log('正在生成一个QA', dataItem._id);
+    console.log('正在生成一个QA, ID:', dataItem._id, 'temperature: ', dataItem.temperature / 100);
+
     const startTime = Date.now();
 
     // 获取 openai 请求实例
@@ -65,7 +66,7 @@ export async function generateQA(next = false): Promise<any> {
     const response = await chatAPI.createChatCompletion(
       {
         model: ChatModelNameEnum.GPT35,
-        temperature: dataItem.temperature,
+        temperature: dataItem.temperature / 100,
         n: 1,
         messages: [
           systemPrompt,
@@ -76,28 +77,27 @@ export async function generateQA(next = false): Promise<any> {
         ]
       },
       {
-        timeout: 60000,
+        timeout: 120000,
         httpsAgent
       }
     );
     const content = response.data.choices[0].message?.content;
     // 从 content 中提取 QA
     const splitResponse = splitText(content || '');
-    if (splitResponse.length > 0) {
+    // 插入数据库，并修改状态
-      // 插入数据库，并修改状态
+    await DataItem.findByIdAndUpdate(dataItem._id, {
-      await DataItem.findByIdAndUpdate(dataItem._id, {
+      status: dataItem.temperature >= 100 ? 0 : 1,
-        status: 0,
+      temperature: dataItem.temperature >= 100 ? dataItem.temperature : dataItem.temperature + 25,
-        $push: {
+      $push: {
-          result: {
+        result: {
-            $each: splitResponse
+          $each: splitResponse
-          }
         }
-      });
+      }
-      console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`);
+    });
-    }
+    console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`);
   } catch (error: any) {
     console.log('error: 生成QA错误', dataItem?._id);
-    console.log('statusText:', error?.response?.statusText);
+    console.log('response:', error?.response);
     // 重置状态
     if (dataItem?._id) {
       await DataItem.findByIdAndUpdate(dataItem._id, {

src/service/models/bill.ts

@@ -1,5 +1,5 @@
 import { Schema, model, models } from 'mongoose';
-import { ModelList } from '@/constants/model';
+import { modelList } from '@/constants/model';
 
 const BillSchema = new Schema({
   userId: {
@@ -14,7 +14,7 @@ const BillSchema = new Schema({
   },
   modelName: {
     type: String,
-    enum: ModelList.map((item) => item.model),
+    enum: modelList.map((item) => item.model),
     required: true
   },
   chatId: {

src/types/data.d.ts

@@ -0,0 +1,6 @@
+import type { DataSchema } from './mongoSchema';
+
+export interface DataListItem extends DataSchema {
+  trainingData: number;
+  totalData: number;
+}

src/types/index.d.ts

@@ -4,6 +4,9 @@ declare global {
   var mongodb: Mongoose | string | null;
   var generatingQA: boolean;
   var QRCode: any;
+  interface Window {
+    ['pdfjs-dist/build/pdf']: any;
+  }
 }
 
 export type PagingData<T> = {

src/utils/tools.ts

@@ -1,5 +1,6 @@
 import crypto from 'crypto';
 import { useToast } from '@/hooks/useToast';
+import mammoth from 'mammoth';
 
 /**
  * copy text data
@@ -50,17 +51,76 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
 };
 
 /**
- * 读取文件内容
+ * 读取 txt 文件内容
  */
-export const loadLocalFileContent = (file: File) => {
+export const readTxtContent = (file: File) => {
   return new Promise((resolve: (_: string) => void, reject) => {
     const reader = new FileReader();
     reader.onload = () => {
       resolve(reader.result as string);
     };
     reader.onerror = (err) => {
-      reject(err);
+      console.log('error txt read:', err);
+      reject('读取 txt 文件失败');
     };
     reader.readAsText(file);
   });
 };
+
+/**
+ * 读取 pdf 内容
+ */
+export const readPdfContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    const pdfjsLib = window['pdfjs-dist/build/pdf'];
+    pdfjsLib.workerSrc = '/js/pdf.worker.js';
+
+    const readPDFPage = async (doc: any, pageNo: number) => {
+      const page = await doc.getPage(pageNo);
+      const tokenizedText = await page.getTextContent();
+      const pageText = tokenizedText.items.map((token: any) => token.str).join('');
+      return pageText.replaceAll(/\s+/g, '\n');
+    };
+
+    let reader = new FileReader();
+    reader.readAsArrayBuffer(file);
+    reader.onload = async (event) => {
+      if (!event?.target?.result) return reject('解析 PDF 失败');
+      try {
+        const doc = await pdfjsLib.getDocument(event.target.result).promise;
+        const pageTextPromises = [];
+        for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
+          pageTextPromises.push(readPDFPage(doc, pageNo));
+        }
+        const pageTexts = await Promise.all(pageTextPromises);
+        resolve(pageTexts.join('\n'));
+      } catch (err) {
+        console.log(err, 'pdfjs error');
+        reject('解析 PDF 失败');
+      }
+    };
+    reader.onerror = (err) => {
+      console.log(err, 'reader error');
+      reject('解析 PDF 失败');
+    };
+  });
+
+/**
+ * 读取doc
+ */
+export const readDocContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsArrayBuffer(file);
+    reader.onload = ({ target }) => {
+      if (!target?.result) return reject('读取 doc 文件失败');
+      return mammoth.extractRawText({ arrayBuffer: target.result as ArrayBuffer }).then((res) => {
+        resolve(res.value);
+      });
+    };
+    reader.onerror = (err) => {
+      console.log('error doc read:', err);
+
+      reject('读取 doc 文件失败');
+    };
+  });