ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-02 12:48:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

External dataset (#1497)

Browse Source 
* perf: read rawText and chunk code

* perf: read raw text

* perf: read rawtext

* perf: token count

* log
This commit is contained in:
Archer
2024-05-16 11:47:53 +08:00
committed by GitHub
parent d5073f98ab
commit c6d9b15897
36 changed files with 531 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
projects/app/src/global/core/dataset/api.d.ts vendored
View File

@@ -1,6 +1,7 @@
import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
import {
DatasetSearchModeEnum,
DatasetSourceReadTypeEnum,
DatasetTypeEnum,
ImportDataSourceEnum,
TrainingModeEnum
@@ -75,22 +76,3 @@ export type SearchTestResponse = {
};
/* =========== training =========== */
export type PostPreviewFilesChunksProps = {
type: ImportDataSourceEnum;
sourceId: string;
chunkSize: number;
overlapRatio: number;
customSplitChar?: string;
};
export type PostPreviewFilesChunksResponse = {
fileId: string;
rawTextLength: number;
chunks: string[];
}[];
export type PostPreviewTableChunksResponse = {
fileId: string;
totalChunks: number;
chunks: { q: string; a: string; chunkIndex: number }[];
errorText?: string;
}[];