Change embedding (#1428)

* fix: text spliter * perf: embedding model
2025-07-29 09:44:47 +00:00 · 2024-05-09 23:23:49 +08:00
parent 434af56abd
commit 5e250b2f65
6 changed files with 93 additions and 11 deletions
--- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
+++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
@@ -35,9 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
          csvFormat: true
        });
        // split chunks (5 chunk)
-        const sliceRawText = 10 * chunkSize;
        const { chunks } = splitText2Chunks({
-          text: rawText.slice(0, sliceRawText),
+          text: rawText,
          chunkLen: chunkSize,
          overlapRatio,
          customReg: customSplitChar ? [customSplitChar] : []