Change embedding (#1428)

* fix: text spliter

* perf: embedding model

projects/app/data/config.json

@@ -80,6 +80,31 @@
     }
   ],
   "vectorModels": [
+    {
+      "model": "text-embedding-3-large",
+      "name": "Embedding-2",
+      "avatar": "/imgs/model/openai.svg",
+      "charsPointsPrice": 0,
+      "defaultToken": 512,
+      "maxToken": 3000,
+      "weight": 100,
+      "dbConfig": {},
+      "queryConfig": {},
+      "defaultConfig": {
+        "dimensions": 1024
+      }
+    },
+    {
+      "model": "text-embedding-3-small",
+      "name": "Embedding-2",
+      "avatar": "/imgs/model/openai.svg",
+      "charsPointsPrice": 0,
+      "defaultToken": 512,
+      "maxToken": 3000,
+      "weight": 100,
+      "dbConfig": {},
+      "queryConfig": {}
+    },
     {
       "model": "text-embedding-ada-002",
       "name": "Embedding-2",

projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts

@@ -35,9 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
           csvFormat: true
         });
         // split chunks (5 chunk)
-        const sliceRawText = 10 * chunkSize;
         const { chunks } = splitText2Chunks({
-          text: rawText.slice(0, sliceRawText),
+          text: rawText,
           chunkLen: chunkSize,
           overlapRatio,
           customReg: customSplitChar ? [customSplitChar] : []