ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 13:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

External dataset (#1497)

Browse Source 
* perf: read rawText and chunk code

* perf: read raw text

* perf: read rawtext

* perf: token count

* log
This commit is contained in:
Archer
2024-05-16 11:47:53 +08:00
committed by GitHub
parent d5073f98ab
commit c6d9b15897
36 changed files with 531 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/service/common/file/gridfs/controller.ts
View File

@@ -151,12 +151,12 @@ export const readFileContentFromMongo = async ({
teamId,
bucketName,
fileId,
csvFormat = false
isQAImport = false
}: {
teamId: string;
bucketName: `${BucketNameEnum}`;
fileId: string;
csvFormat?: boolean;
isQAImport?: boolean;
}): Promise<{
rawText: string;
filename: string;
@@ -198,7 +198,7 @@ export const readFileContentFromMongo = async ({
const { rawText } = await readFileRawContent({
extension,
csvFormat,
isQAImport,
teamId,
buffer: fileBuffers,
encoding,