v4.6-4 (#473)

packages/global/common/string/textSplitter.ts

@@ -63,8 +63,8 @@ export const splitText2Chunks = (props: { text: string; maxLen: number; overlapL
     let chunks: string[] = [];
     for (let i = 0; i < splitTexts.length; i++) {
       let text = splitTexts[i];
-      let chunkToken = countPromptTokens(lastChunk, '');
-      const textToken = countPromptTokens(text, '');
+      let chunkToken = lastChunk.length;
+      const textToken = text.length;
 
       // next chunk is too large / new chunk is too large(The current chunk must be smaller than maxLen)
       if (textToken >= maxLen || chunkToken + textToken > maxLen * 1.4) {

packages/global/core/dataset/type.d.ts

@@ -1,4 +1,4 @@
-import type { VectorModelItemType } from '../../core/ai/model.d';
+import type { LLMModelItemType, VectorModelItemType } from '../../core/ai/model.d';
 import { PermissionTypeEnum } from '../../support/permission/constant';
 import { PushDatasetDataChunkProps } from './api';
 import {
@@ -19,6 +19,7 @@ export type DatasetSchemaType = {
   avatar: string;
   name: string;
   vectorModel: string;
+  agentModel: string;
   tags: string[];
   type: `${DatasetTypeEnum}`;
   permission: `${PermissionTypeEnum}`;
@@ -84,8 +85,9 @@ export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datas
 };
 
 /* ================= dataset ===================== */
-export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel'> & {
+export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
   vectorModel: VectorModelItemType;
+  agentModel: LLMModelItemType;
   isOwner: boolean;
   canWrite: boolean;
 };

packages/global/support/wallet/bill/api.d.ts

@@ -3,6 +3,8 @@ import { BillListItemType } from './type';
 
 export type CreateTrainingBillProps = {
   name: string;
+  vectorModel?: string;
+  agentModel?: string;
 };
 
 export type ConcatBillProps = {