feat: chunk index independent config (#4271)

* sync collection

* remove lock

* feat: chunk index independent config

* feat: add max chunksize to split chunk function

* remove log

* update doc

* remove

* remove log

packages/service/core/dataset/collection/controller.ts

@@ -27,6 +27,11 @@ import { addDays } from 'date-fns';
 import { MongoDatasetDataText } from '../data/dataTextSchema';
 import { retryFn } from '@fastgpt/global/common/system/utils';
 import { getTrainingModeByCollection } from './utils';
+import {
+  computeChunkSize,
+  computeChunkSplitter,
+  getLLMMaxChunkSize
+} from '@fastgpt/global/core/dataset/training/utils';
 
 export const createCollectionAndInsertData = async ({
   dataset,
@@ -54,18 +59,22 @@ export const createCollectionAndInsertData = async ({
 
   const teamId = createCollectionParams.teamId;
   const tmbId = createCollectionParams.tmbId;
-  // Chunk split params
+
+  // Set default params
   const trainingType =
     createCollectionParams.trainingType || DatasetCollectionDataProcessModeEnum.chunk;
-  const chunkSize = createCollectionParams.chunkSize || 512;
-  const chunkSplitter = createCollectionParams.chunkSplitter;
-  const qaPrompt = createCollectionParams.qaPrompt;
-  const usageName = createCollectionParams.name;
+  const chunkSize = computeChunkSize({
+    ...createCollectionParams,
+    trainingType,
+    llmModel: getLLMModel(dataset.agentModel)
+  });
+  const chunkSplitter = computeChunkSplitter(createCollectionParams);
 
   // 1. split chunks
   const chunks = rawText2Chunks({
     rawText,
-    chunkLen: chunkSize,
+    chunkSize,
+    maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)),
     overlapRatio: trainingType === DatasetCollectionDataProcessModeEnum.chunk ? 0.2 : 0,
     customReg: chunkSplitter ? [chunkSplitter] : [],
     isQAImport
@@ -76,7 +85,7 @@ export const createCollectionAndInsertData = async ({
     teamId,
     insertLen: predictDataLimitLength(
       getTrainingModeByCollection({
-        trainingType,
+        trainingType: trainingType,
         autoIndexes: createCollectionParams.autoIndexes,
         imageIndex: createCollectionParams.imageIndex
       }),
@@ -88,6 +97,9 @@ export const createCollectionAndInsertData = async ({
     // 3. create collection
     const { _id: collectionId } = await createOneCollection({
       ...createCollectionParams,
+      trainingType,
+      chunkSize,
+      chunkSplitter,
 
       hashRawText: hashStr(rawText),
       rawTextLength: rawText.length,
@@ -111,7 +123,7 @@ export const createCollectionAndInsertData = async ({
       const { billId: newBillId } = await createTrainingUsage({
         teamId,
         tmbId,
-        appName: usageName,
+        appName: createCollectionParams.name,
         billSource: UsageSourceEnum.training,
         vectorModel: getEmbeddingModel(dataset.vectorModel)?.name,
         agentModel: getLLMModel(dataset.agentModel)?.name,
@@ -130,12 +142,13 @@ export const createCollectionAndInsertData = async ({
       agentModel: dataset.agentModel,
       vectorModel: dataset.vectorModel,
       vlmModel: dataset.vlmModel,
+      indexSize: createCollectionParams.indexSize,
       mode: getTrainingModeByCollection({
-        trainingType,
+        trainingType: trainingType,
         autoIndexes: createCollectionParams.autoIndexes,
         imageIndex: createCollectionParams.imageIndex
       }),
-      prompt: qaPrompt,
+      prompt: createCollectionParams.qaPrompt,
       billId: traingBillId,
       data: chunks.map((item, index) => ({
         ...item,
@@ -207,11 +220,14 @@ export async function createOneCollection({
   // Parse settings
   customPdfParse,
   imageIndex,
+  autoIndexes,
 
   // Chunk settings
-  trainingType = DatasetCollectionDataProcessModeEnum.chunk,
-  autoIndexes,
-  chunkSize = 512,
+  trainingType,
+  chunkSettingMode,
+  chunkSplitMode,
+  chunkSize,
+  indexSize,
   chunkSplitter,
   qaPrompt,
 
@@ -249,11 +265,14 @@ export async function createOneCollection({
         // Parse settings
         customPdfParse,
         imageIndex,
+        autoIndexes,
 
         // Chunk settings
         trainingType,
-        autoIndexes,
+        chunkSettingMode,
+        chunkSplitMode,
         chunkSize,
+        indexSize,
         chunkSplitter,
         qaPrompt
       }

packages/service/core/dataset/collection/schema.ts

@@ -3,7 +3,9 @@ const { Schema, model, models } = connectionMongo;
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import {
   DatasetCollectionTypeMap,
-  DatasetCollectionDataProcessModeEnum
+  DatasetCollectionDataProcessModeEnum,
+  ChunkSettingModeEnum,
+  DataChunkSplitModeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import { DatasetCollectionName } from '../schema';
 import {
@@ -94,11 +96,18 @@ const DatasetCollectionSchema = new Schema({
     type: String,
     enum: Object.values(DatasetCollectionDataProcessModeEnum)
   },
-  chunkSize: {
-    type: Number,
-    required: true
+  chunkSettingMode: {
+    type: String,
+    enum: Object.values(ChunkSettingModeEnum)
   },
+  chunkSplitMode: {
+    type: String,
+    enum: Object.values(DataChunkSplitModeEnum)
+  },
+  chunkSize: Number,
   chunkSplitter: String,
+
+  indexSize: Number,
   qaPrompt: String
 });