Perf: read file woker (#1337)

* perf: read file worker

* fix: Http node url input

* fix: htm2md

* fix: html2md

* fix: ts

* perf: Problem classification increases the matching order

* feat: tool response answer
This commit is contained in:
Archer
2024-04-30 18:12:20 +08:00
committed by GitHub
parent 1529c1e991
commit b5f0ac3e1d
35 changed files with 413 additions and 398 deletions

View File

@@ -6,7 +6,6 @@ import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
import { MongoFileSchema } from './schema';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { ReadFileByBufferParams } from '../read/type';
import { MongoRwaTextBuffer } from '../../buffer/rawText/schema';
import { readFileRawContent } from '../read/utils';
import { PassThrough } from 'stream';
@@ -197,19 +196,15 @@ export const readFileContentFromMongo = async ({
});
})();
const params: ReadFileByBufferParams = {
const { rawText } = await readFileRawContent({
extension,
csvFormat,
teamId,
buffer: fileBuffers,
encoding,
metadata: {
relatedId: fileId
}
};
const { rawText } = await readFileRawContent({
extension,
csvFormat,
params
});
if (rawText.trim()) {