ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-30 02:12:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: password special chars;feat: llm paragraph;perf: chunk setting params;perf: text splitter worker (#4984)

Browse Source 
* perf: password special chars

* feat: llm paragraph;perf: chunk setting params

* perf: text splitter worker

* perf: get rawtext buffer

* fix: test

* fix: test

* doc

* min chunk size
This commit is contained in:
Archer
2025-06-10 00:05:54 +08:00
committed by GitHub
parent 068918a9ee
commit 01ff56b42b
41 changed files with 546 additions and 448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/service/common/file/gridfs/utils.ts
View File

@@ -55,13 +55,17 @@ export const createFileFromText = async ({
export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
return new Promise<Buffer>((resolve, reject) => {
if (!stream.readable) {
return resolve(Buffer.from([]));
}
const chunks: Uint8Array[] = [];
stream.on('data', (chunk) => {
chunks.push(chunk);
});
stream.on('end', () => {
const resultBuffer = Buffer.concat(chunks); // 一次性拼接
const resultBuffer = Buffer.concat(chunks); // One-time splicing
resolve(resultBuffer);
});
stream.on('error', (err) => {