ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 13:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updae theme and fix some bug (#1711)

Browse Source
This commit is contained in:
Archer
2024-06-07 12:54:30 +08:00
committed by GitHub
parent 19c8a06d51
commit b20d075d35
153 changed files with 1587 additions and 1436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
packages/service/common/file/gridfs/controller.ts
View File

@@ -8,7 +8,7 @@ import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
import { readRawContentByFileBuffer } from '../read/utils';
import { PassThrough } from 'stream';
import { gridFsStream2Buffer } from './utils';
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
MongoFileSchema;
@@ -113,35 +113,16 @@ export async function getDownloadStream({
fileId: string;
}) {
const bucket = getGridBucket(bucketName);
const stream = bucket.openDownloadStream(new Types.ObjectId(fileId));
const copyStream = stream.pipe(new PassThrough());
const encodeStream = bucket.openDownloadStream(new Types.ObjectId(fileId), { end: 100 });
const rawStream = bucket.openDownloadStream(new Types.ObjectId(fileId));
/* get encoding */
const buffer = await (() => {
return new Promise<Buffer>((resolve, reject) => {
let tmpBuffer: Buffer = Buffer.from([]);
stream.on('data', (chunk) => {
if (tmpBuffer.length < 20) {
tmpBuffer = Buffer.concat([tmpBuffer, chunk]);
}
if (tmpBuffer.length >= 20) {
resolve(tmpBuffer);
}
});
stream.on('end', () => {
resolve(tmpBuffer);
});
stream.on('error', (err) => {
reject(err);
});
});
})();
const buffer = await gridFsStream2Buffer(encodeStream);
const encoding = detectFileEncoding(buffer);
return {
fileStream: copyStream,
fileStream: rawStream,
encoding
// encoding: 'utf-8'
};
@@ -169,32 +150,21 @@ export const readFileContentFromMongo = async ({
filename: fileBuffer.metadata?.filename || ''
};
}
const start = Date.now();
const [file, { encoding, fileStream }] = await Promise.all([
getFileById({ bucketName, fileId }),
getDownloadStream({ bucketName, fileId })
]);
// console.log('get file stream', Date.now() - start);
if (!file) {
return Promise.reject(CommonErrEnum.fileNotFound);
}
const extension = file?.filename?.split('.')?.pop()?.toLowerCase() || '';
const fileBuffers = await (() => {
return new Promise<Buffer>((resolve, reject) => {
let buffer = Buffer.from([]);
fileStream.on('data', (chunk) => {
buffer = Buffer.concat([buffer, chunk]);
});
fileStream.on('end', () => {
resolve(buffer);
});
fileStream.on('error', (err) => {
reject(err);
});
});
})();
const fileBuffers = await gridFsStream2Buffer(fileStream);
// console.log('get file buffer', Date.now() - start);
const { rawText } = await readRawContentByFileBuffer({
extension,

15
packages/service/common/file/gridfs/utils.ts Normal file
View File

@@ -0,0 +1,15 @@
export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
return new Promise<Buffer>((resolve, reject) => {
let tmpBuffer: Buffer = Buffer.from([]);
stream.on('data', (chunk) => {
tmpBuffer = Buffer.concat([tmpBuffer, chunk]);
});
stream.on('end', () => {
resolve(tmpBuffer);
});
stream.on('error', (err) => {
reject(err);
});
});
};