perf: password special chars;feat: llm paragraph;perf: chunk setting params;perf: text splitter worker (#4984)

* perf: password special chars

* feat: llm paragraph;perf: chunk setting params

* perf: text splitter worker

* perf: get rawtext buffer

* fix: test

* fix: test

* doc

* min chunk size
This commit is contained in:
Archer
2025-06-10 00:05:54 +08:00
committed by GitHub
parent 068918a9ee
commit 01ff56b42b
41 changed files with 546 additions and 448 deletions

View File

@@ -1,3 +1,4 @@
import { isTestEnv } from '@fastgpt/global/common/system/constants';
import { addLog } from '../../common/system/log';
import type { Model } from 'mongoose';
import mongoose, { Mongoose } from 'mongoose';
@@ -70,7 +71,7 @@ const addCommonMiddleware = (schema: mongoose.Schema) => {
export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
if (connectionMongo.models[name]) return connectionMongo.models[name] as Model<T>;
if (process.env.NODE_ENV !== 'test') console.log('Load model======', name);
if (!isTestEnv) console.log('Load model======', name);
addCommonMiddleware(schema);
const model = connectionMongo.model<T>(name, schema);