ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 13:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: worker un exit (#624)

Browse Source
This commit is contained in:
Archer
2023-12-18 20:35:17 +08:00
committed by GitHub
parent 703583fff7
commit b14a1db2f9
9 changed files with 58 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/global/common/string/textSplitter.ts
View File

@@ -232,7 +232,7 @@ export const splitText2Chunks = (props: {
step: 0,
lastText: '',
mdTitle: ''
}).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block
}).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block
const tokens = countTokens
? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0)

3
packages/service/common/string/markdown.ts
View File

@@ -13,9 +13,12 @@ export const htmlToMarkdown = (html?: string | null) =>
const worker = new Worker(getWorkerPath('html2md'));
worker.on('message', (md: string) => {
worker.terminate();
resolve(simpleMarkdownText(md));
});
worker.on('error', (err) => {
worker.terminate();
reject(err);
});

35
pnpm-lock.yaml generated
View File

@@ -84,15 +84,9 @@ importers:
dayjs:
specifier: ^1.11.7
version: registry.npmmirror.com/dayjs@1.11.7
domino-ext:
specifier: ^2.1.4
version: registry.npmmirror.com/domino-ext@2.1.4
encoding:
specifier: ^0.1.13
version: registry.npmmirror.com/encoding@0.1.13
joplin-turndown-plugin-gfm:
specifier: ^1.0.12
version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12
jsonwebtoken:
specifier: ^9.0.2
version: registry.npmmirror.com/jsonwebtoken@9.0.2
@@ -117,9 +111,6 @@ importers:
tunnel:
specifier: ^0.0.6
version: registry.npmmirror.com/tunnel@0.0.6
turndown:
specifier: ^7.1.2
version: registry.npmmirror.com/turndown@7.1.2
devDependencies:
'@types/cookie':
specifier: ^0.5.2
@@ -6765,18 +6756,6 @@ packages:
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
dev: false
registry.npmmirror.com/domino-ext@2.1.4:
resolution: {integrity: sha512-t8piRI9Qahd4V/NqnCcqdBWsQ4OYeOvcTuoHl38Pzk9OJJ/UiCYHA2jX2fACmBtDlSMiWa0uR524KuLEAMc/3Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino-ext/-/domino-ext-2.1.4.tgz}
name: domino-ext
version: 2.1.4
dev: false
registry.npmmirror.com/domino@2.1.6:
resolution: {integrity: sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino/-/domino-2.1.6.tgz}
name: domino
version: 2.1.6
dev: false
registry.npmmirror.com/dompurify@3.0.3:
resolution: {integrity: sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.3.tgz}
name: dompurify
@@ -8793,12 +8772,6 @@ packages:
set-function-name: registry.npmmirror.com/set-function-name@2.0.1
dev: true
registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12:
resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/joplin-turndown-plugin-gfm/-/joplin-turndown-plugin-gfm-1.0.12.tgz}
name: joplin-turndown-plugin-gfm
version: 1.0.12
dev: false
registry.npmmirror.com/js-sdsl@4.4.2:
resolution: {integrity: sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-sdsl/-/js-sdsl-4.4.2.tgz}
name: js-sdsl
@@ -12193,14 +12166,6 @@ packages:
engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'}
dev: false
registry.npmmirror.com/turndown@7.1.2:
resolution: {integrity: sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/turndown/-/turndown-7.1.2.tgz}
name: turndown
version: 7.1.2
dependencies:
domino: registry.npmmirror.com/domino@2.1.6
dev: false
registry.npmmirror.com/type-check@0.4.0:
resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/type-check/-/type-check-0.4.0.tgz}
name: type-check

2
projects/app/src/components/Markdown/chat/Guide.tsx
View File

@@ -23,7 +23,7 @@ function MyLink(e: any) {
<Box as={'li'} mb={1}>
<Box
as={'span'}
color={'blue.500'}
color={'blue.700'}
textDecoration={'underline'}
cursor={'pointer'}
onClick={() => {

2
projects/app/src/components/SideTabs/index.tsx
View File

@@ -44,7 +44,7 @@ const SideTabs = ({ list, size = 'md', activeId, onChange, ...props }: Props) =>
alignItems={'center'}
{...(activeId === item.id
? {
bg: ' blue.200 !important',
bg: ' blue.100 !important',
fontWeight: 'bold',
color: 'blue.600 ',
cursor: 'default'

8
projects/app/src/components/core/module/Flow/components/modules/ButtonEdge.tsx
View File

@@ -1,11 +1,5 @@
import React from 'react';
import {
SmoothStepEdge,
EdgeLabelRenderer,
EdgeProps,
getSmoothStepPath,
MarkerType
} from 'reactflow';
import { SmoothStepEdge, EdgeLabelRenderer, EdgeProps, getSmoothStepPath } from 'reactflow';
import { Flex } from '@chakra-ui/react';
import MyIcon from '@/components/Icon';

84
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
View File

@@ -289,28 +289,30 @@ export const PreviewFileOrChunk = () => {
overflow={'overlay'}
px={[4, 8]}
my={4}
contentEditable
dangerouslySetInnerHTML={{ __html: previewFile.rawText }}
// contentEditable
// dangerouslySetInnerHTML={{ __html: previewFile.rawText }}
fontSize={'sm'}
whiteSpace={'pre-wrap'}
wordBreak={'break-all'}
onBlur={(e) => {
// @ts-ignore
const val = e.target.innerText;
setReShowRePreview(true);
// onBlur={(e) => {
// // @ts-ignore
// const val = e.target.innerText;
// setReShowRePreview(true);
setFiles((state) =>
state.map((file) =>
file.id === previewFile.id
? {
...file,
text: val
}
: file
)
);
}}
/>
// setFiles((state) =>
// state.map((file) =>
// file.id === previewFile.id
// ? {
// ...file,
// text: val
// }
// : file
// )
// );
// }}
>
{previewFile.rawText}
</Box>
</Box>
) : (
<Box pt={[3, 6]}>
@@ -365,49 +367,9 @@ export const PreviewFileOrChunk = () => {
}}
/>
</Flex>
<Box
px={4}
fontSize={'sm'}
whiteSpace={'pre-wrap'}
wordBreak={'break-all'}
contentEditable={!chunk.a}
dangerouslySetInnerHTML={{
__html: chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q
}}
onBlur={(e) => {
// @ts-ignore
const val = e.target.innerText;
/* delete file */
if (val === '') {
setFiles((state) =>
state.map((stateFile) =>
stateFile.id === file.id
? {
...file,
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
}
: stateFile
)
);
} else {
// update chunk
setFiles((stateFiles) =>
stateFiles.map((stateFile) =>
file.id === stateFile.id
? {
...stateFile,
chunks: stateFile.chunks.map((chunk, index) => ({
...chunk,
index: i === index ? val : chunk.q
}))
}
: stateFile
)
);
}
}}
/>
<Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
{chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q}
</Box>
</Box>
))
)}

20
projects/app/src/web/styles/theme.ts
View File

@@ -284,16 +284,16 @@ export const theme = extendTheme({
1000: '#121416'
},
blue: {
50: '#f0f7ff',
100: '#EBF7FD',
200: '#d6e8ff',
300: '#adceff',
400: '#85b1ff',
500: '#4e83fd',
600: '#3370ff',
700: '#2152d9',
800: '#1237b3',
900: '#07228c'
50: '#F0F4FF',
100: '#E1EAFF',
200: '#C5D7FF',
300: '#94B5FF',
400: '#5E8FFF',
500: '#487FFF',
600: '#3370FF',
700: '#2B5FD9',
800: '#2450B5',
900: '#1D4091'
},
myRead: {
600: '#ff4d4f'

32
worker/html2md.js
View File

@@ -20,22 +20,26 @@ parentPort?.on('message', (html) => {
});
const html2md = (html) => {
const window = domino.createWindow(html);
const document = window.document;
try {
const window = domino.createWindow(html);
const document = window.document;
turndownService.remove(['i', 'script', 'iframe']);
turndownService.addRule('codeBlock', {
filter: 'pre',
replacement(_, node) {
const content = node.textContent?.trim() || '';
// @ts-ignore
const codeName = node?._attrsByQName?.class?.data?.trim() || '';
turndownService.remove(['i', 'script', 'iframe']);
turndownService.addRule('codeBlock', {
filter: 'pre',
replacement(_, node) {
const content = node.textContent?.trim() || '';
// @ts-ignore
const codeName = node?._attrsByQName?.class?.data?.trim() || '';
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
}
});
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
}
});
turndownService.use(turndownPluginGfm.gfm);
turndownService.use(turndownPluginGfm.gfm);
return turndownService.turndown(document);
return turndownService.turndown(document);
} catch (error) {
return '';
}
};