v4.6.5 (#620)

worker/html2md.js

@@ -0,0 +1,41 @@
+const { parentPort } = require('worker_threads');
+const TurndownService = require('turndown');
+const domino = require('domino-ext');
+const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
+
+const turndownService = new TurndownService({
+  headingStyle: 'atx',
+  bulletListMarker: '-',
+  codeBlockStyle: 'fenced',
+  fence: '```',
+  emDelimiter: '_',
+  strongDelimiter: '**',
+  linkStyle: 'inlined',
+  linkReferenceStyle: 'full'
+});
+parentPort?.on('message', (html) => {
+  const md = html2md(html);
+
+  parentPort.postMessage(md);
+});
+
+const html2md = (html) => {
+  const window = domino.createWindow(html);
+  const document = window.document;
+
+  turndownService.remove(['i', 'script', 'iframe']);
+  turndownService.addRule('codeBlock', {
+    filter: 'pre',
+    replacement(_, node) {
+      const content = node.textContent?.trim() || '';
+      // @ts-ignore
+      const codeName = node?._attrsByQName?.class?.data?.trim() || '';
+
+      return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
+    }
+  });
+
+  turndownService.use(turndownPluginGfm.gfm);
+
+  return turndownService.turndown(document);
+};

worker/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@fastgpt/worker",
+  "version": "1.0.0",
+  "dependencies": {
+    "domino-ext": "^2.1.4",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
+    "turndown": "^7.1.2"
+  },
+  "devDependencies": {}
+}

worker/pnpm-lock.yaml

@@ -0,0 +1,44 @@
+lockfileVersion: '6.1'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+dependencies:
+  domino-ext:
+    specifier: ^2.1.4
+    version: registry.npmmirror.com/domino-ext@2.1.4
+  joplin-turndown-plugin-gfm:
+    specifier: ^1.0.12
+    version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12
+  turndown:
+    specifier: ^7.1.2
+    version: registry.npmmirror.com/turndown@7.1.2
+
+packages:
+
+  registry.npmmirror.com/domino-ext@2.1.4:
+    resolution: {integrity: sha512-t8piRI9Qahd4V/NqnCcqdBWsQ4OYeOvcTuoHl38Pzk9OJJ/UiCYHA2jX2fACmBtDlSMiWa0uR524KuLEAMc/3Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino-ext/-/domino-ext-2.1.4.tgz}
+    name: domino-ext
+    version: 2.1.4
+    dev: false
+
+  registry.npmmirror.com/domino@2.1.6:
+    resolution: {integrity: sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino/-/domino-2.1.6.tgz}
+    name: domino
+    version: 2.1.6
+    dev: false
+
+  registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12:
+    resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/joplin-turndown-plugin-gfm/-/joplin-turndown-plugin-gfm-1.0.12.tgz}
+    name: joplin-turndown-plugin-gfm
+    version: 1.0.12
+    dev: false
+
+  registry.npmmirror.com/turndown@7.1.2:
+    resolution: {integrity: sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/turndown/-/turndown-7.1.2.tgz}
+    name: turndown
+    version: 7.1.2
+    dependencies:
+      domino: registry.npmmirror.com/domino@2.1.6
+    dev: false