Files
FastGPT/packages/service/worker/readFile/index.ts
Archer 7bcee82f5f perf: memory leak (#5370)
* perf: memory leak

* perf: workflow share buffer;Circle checker;Get file from stream

* doc

* remove report.md
2025-08-03 22:37:45 +08:00

69 lines
2.0 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { parentPort } from 'worker_threads';
import { readFileRawText } from './extension/rawText';
import { type ReadRawTextByBuffer, type ReadRawTextProps } from './type';
import { readHtmlRawText } from './extension/html';
import { readPdfFile } from './extension/pdf';
import { readDocsFile } from './extension/docx';
import { readPptxRawText } from './extension/pptx';
import { readXlsxRawText } from './extension/xlsx';
import { readCsvRawText } from './extension/csv';
import { workerResponse } from '../controller';
parentPort?.on(
'message',
async (
props: Omit<ReadRawTextProps<any>, 'buffer'> & {
sharedBuffer: SharedArrayBuffer;
bufferSize: number;
}
) => {
const read = async (params: ReadRawTextByBuffer) => {
switch (params.extension) {
case 'txt':
case 'md':
return readFileRawText(params);
case 'html':
return readHtmlRawText(params);
case 'pdf':
return readPdfFile(params);
case 'docx':
return readDocsFile(params);
case 'pptx':
return readPptxRawText(params);
case 'xlsx':
return readXlsxRawText(params);
case 'csv':
return readCsvRawText(params);
default:
return Promise.reject(
`Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx. "${params.extension}" is not supported.`
);
}
};
// 使用 SharedArrayBuffer零拷贝共享内存
const sharedArray = new Uint8Array(props.sharedBuffer);
const buffer = Buffer.from(sharedArray.buffer, 0, props.bufferSize);
const newProps: ReadRawTextByBuffer = {
extension: props.extension,
encoding: props.encoding,
buffer
};
try {
workerResponse({
parentPort,
status: 'success',
data: await read(newProps)
});
} catch (error) {
workerResponse({
parentPort,
status: 'error',
data: error
});
}
}
);