update text splitter (#3020)

projects/app/src/pages/api/admin/resetMilvus.ts

@@ -54,7 +54,6 @@ async function handler(
           }
         );
         dataLength += data.matchedCount;
-        console.log(data.matchedCount, '=-=-');
 
         // 插入数据进入训练库
         const max = global.systemEnv?.vectorMaxProcess || 10;

projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts

@@ -3,7 +3,7 @@ import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { NextAPI } from '@/service/middleware/entry';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
-import { OwnerPermissionVal, ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { authFile } from '@fastgpt/service/support/permission/auth/file';
 
 export type PostPreviewFilesChunksProps = {
@@ -60,6 +60,6 @@ async function handler(
     overlapRatio,
     customReg: customSplitChar ? [customSplitChar] : [],
     isQAImport: isQAImport
-  }).slice(0, 5);
+  }).slice(0, 15);
 }
 export default NextAPI(handler);

projects/app/src/web/common/hooks/useSpeech.ts

@@ -111,7 +111,6 @@ export const useSpeech = (props?: OutLinkChatAuthProps & { appId?: string }) =>
 
           const blob = new Blob(chunks, options);
           const duration = Math.round((Date.now() - startTimestamp.current) / 1000);
-          console.log(options, filename, '=-=-');
           formData.append('file', blob, filename);
           formData.append(
             'data',

projects/sandbox/package.json

@@ -27,7 +27,7 @@
     "fastify": "^4.27.0",
     "dayjs": "^1.11.7",
     "isolated-vm": "^4.7.2",
-    "tiktoken": "^1.0.15",
+    "tiktoken": "1.0.17",
     "node-gyp": "^10.1.0",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1"