perf: text splitter (#4584)

* perf: text splitter

* update doc

projects/app/src/pageComponents/dataset/detail/DataCard.tsx

@@ -31,6 +31,7 @@ import { TabEnum } from './NavBar';
 import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import TrainingStates from './CollectionCard/TrainingStates';
+import { getTextValidLength } from '@fastgpt/global/common/string/utils';
 
 const DataCard = () => {
   const theme = useTheme();
@@ -327,7 +328,7 @@ const DataCard = () => {
                       w={'14px'}
                       mr={1}
                     />
-                    {item.q.length + (item.a?.length || 0)}
+                    {getTextValidLength(item.q + item.a || '')}
                   </Flex>
                   {canWrite && (
                     <IconButton

projects/app/src/service/core/dataset/data/controller.ts

@@ -77,6 +77,7 @@ const formatIndexes = async ({
 
   // Recompute default indexes, Merge ids of the same index, reduce the number of rebuilds
   const defaultIndexes = getDefaultIndex({ q, a, indexSize });
+
   const concatDefaultIndexes = defaultIndexes.map((item) => {
     const oldIndex = indexes!.find((index) => index.text === item.text);
     if (oldIndex) {