Add image index and pdf parse (#3956)

* feat: think tag parse

* feat: parse think tag test

* feat: pdf parse ux

* feat: doc2x parse

* perf: rewrite training mode setting

* feat: image parse queue

* perf: image index

* feat: image parse process

* feat: add init sh

* fix: ts

projects/app/src/pageComponents/account/model/Channel/ModelTest.tsx

@@ -23,7 +23,7 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
 import MyTag from '@fastgpt/web/components/common/Tag/index';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import { batchRun } from '@fastgpt/global/common/fn/utils';
+import { batchRun } from '@fastgpt/global/common/system/utils';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 
 type ModelTestItem = {

projects/app/src/pageComponents/account/usage/UsageDetail.tsx

@@ -26,7 +26,7 @@ const UsageDetail = ({ usage, onClose }: { usage: UsageItemType; onClose: () =>
     [usage.list]
   );
 
-  const { hasModel, hasToken, hasInputToken, hasOutputToken, hasCharsLen, hasDuration } =
+  const { hasModel, hasToken, hasInputToken, hasOutputToken, hasCharsLen, hasDuration, hasPages } =
     useMemo(() => {
       let hasModel = false;
       let hasToken = false;
@@ -34,7 +34,7 @@ const UsageDetail = ({ usage, onClose }: { usage: UsageItemType; onClose: () =>
       let hasOutputToken = false;
       let hasCharsLen = false;
       let hasDuration = false;
-      let hasDataLen = false;
+      let hasPages = false;
 
       usage.list.forEach((item) => {
         if (item.model !== undefined) {
@@ -56,6 +56,9 @@ const UsageDetail = ({ usage, onClose }: { usage: UsageItemType; onClose: () =>
         if (typeof item.duration === 'number') {
           hasDuration = true;
         }
+        if (typeof item.pages === 'number') {
+          hasPages = true;
+        }
       });
 
       return {
@@ -65,7 +68,7 @@ const UsageDetail = ({ usage, onClose }: { usage: UsageItemType; onClose: () =>
         hasOutputToken,
         hasCharsLen,
         hasDuration,
-        hasDataLen
+        hasPages
       };
     }, [usage.list]);
 
@@ -113,6 +116,7 @@ const UsageDetail = ({ usage, onClose }: { usage: UsageItemType; onClose: () =>
                   {hasOutputToken && <Th>{t('account_usage:output_token_length')}</Th>}
                   {hasCharsLen && <Th>{t('account_usage:text_length')}</Th>}
                   {hasDuration && <Th>{t('account_usage:duration_seconds')}</Th>}
+                  {hasPages && <Th>{t('account_usage:pages')}</Th>}
                   <Th>{t('account_usage:total_points_consumed')}</Th>
                 </Tr>
               </Thead>
@@ -126,6 +130,7 @@ const UsageDetail = ({ usage, onClose }: { usage: UsageItemType; onClose: () =>
                     {hasOutputToken && <Td>{item.outputTokens ?? '-'}</Td>}
                     {hasCharsLen && <Td>{item.charsLength ?? '-'}</Td>}
                     {hasDuration && <Td>{item.duration ?? '-'}</Td>}
+                    {hasPages && <Td>{item.pages ?? '-'}</Td>}
                     <Td>{formatNumber(item.amount)}</Td>
                   </Tr>
                 ))}

projects/app/src/pageComponents/account/usage/UsageTable.tsx

@@ -87,8 +87,8 @@ const UsageTableList = ({
               'common:support.wallet.usage.Audio Speech'
             ),
             ['support.wallet.usage.Whisper']: t('common:support.wallet.usage.Whisper'),
-            ['support.wallet.moduleName.index']: t('common:support.wallet.moduleName.index'),
-            ['support.wallet.moduleName.qa']: t('common:support.wallet.moduleName.qa'),
+            ['account_usage:embedding_index']: t('account_usage:embedding_index'),
+            ['account_usage:qa']: t('account_usage:qa'),
             ['core.dataset.training.Auto mode']: t('common:core.dataset.training.Auto mode'),
             ['common:core.module.template.ai_chat']: t('common:core.module.template.ai_chat')
           },
@@ -122,49 +122,51 @@ const UsageTableList = ({
           onConfirm={exportUsage}
         />
       </Flex>
-      <MyBox position={'relative'} overflowY={'auto'} mt={3} flex={1} isLoading={isLoading}>
-        <TableContainer>
-          <Table>
-            <Thead>
-              <Tr>
-                <Th>{t('common:user.Time')}</Th>
-                <Th>{t('account_usage:member')}</Th>
-                <Th>{t('account_usage:user_type')}</Th>
-                <Th>{t('account_usage:project_name')}</Th>
-                <Th>{t('account_usage:total_points')}</Th>
-                <Th></Th>
-              </Tr>
-            </Thead>
-            <Tbody fontSize={'sm'}>
-              {usages.map((item) => (
-                <Tr key={item.id}>
-                  <Td>{dayjs(item.time).format('YYYY/MM/DD HH:mm:ss')}</Td>
-                  <Td>
-                    <Flex alignItems={'center'} color={'myGray.500'}>
-                      <Avatar src={item.sourceMember.avatar} w={'20px'} mr={1} rounded={'full'} />
-                      {item.sourceMember.name}
-                    </Flex>
-                  </Td>
-                  <Td>{t(UsageSourceMap[item.source]?.label as any) || '-'}</Td>
-                  <Td>{t(item.appName as any) || '-'}</Td>
-                  <Td>{formatNumber(item.totalPoints) || 0}</Td>
-                  <Td>
-                    <Button
-                      size={'sm'}
-                      variant={'whitePrimary'}
-                      onClick={() => setUsageDetail(item)}
-                    >
-                      {t('account_usage:details')}
-                    </Button>
-                  </Td>
+      <MyBox mt={3} flex={'1 0 0'} h={0} isLoading={isLoading}>
+        <Box h={'100%'} overflow={'auto'}>
+          <TableContainer>
+            <Table>
+              <Thead>
+                <Tr>
+                  <Th>{t('common:user.Time')}</Th>
+                  <Th>{t('account_usage:member')}</Th>
+                  <Th>{t('account_usage:user_type')}</Th>
+                  <Th>{t('account_usage:project_name')}</Th>
+                  <Th>{t('account_usage:total_points')}</Th>
+                  <Th></Th>
                 </Tr>
-              ))}
-            </Tbody>
-          </Table>
-          {!isLoading && usages.length === 0 && (
-            <EmptyTip text={t('account_usage:no_usage_records')}></EmptyTip>
-          )}
-        </TableContainer>
+              </Thead>
+              <Tbody fontSize={'sm'}>
+                {usages.map((item) => (
+                  <Tr key={item.id}>
+                    <Td>{dayjs(item.time).format('YYYY/MM/DD HH:mm:ss')}</Td>
+                    <Td>
+                      <Flex alignItems={'center'} color={'myGray.500'}>
+                        <Avatar src={item.sourceMember.avatar} w={'20px'} mr={1} rounded={'full'} />
+                        {item.sourceMember.name}
+                      </Flex>
+                    </Td>
+                    <Td>{t(UsageSourceMap[item.source]?.label as any) || '-'}</Td>
+                    <Td>{t(item.appName as any) || '-'}</Td>
+                    <Td>{formatNumber(item.totalPoints) || 0}</Td>
+                    <Td>
+                      <Button
+                        size={'sm'}
+                        variant={'whitePrimary'}
+                        onClick={() => setUsageDetail(item)}
+                      >
+                        {t('account_usage:details')}
+                      </Button>
+                    </Td>
+                  </Tr>
+                ))}
+              </Tbody>
+            </Table>
+            {!isLoading && usages.length === 0 && (
+              <EmptyTip text={t('account_usage:no_usage_records')}></EmptyTip>
+            )}
+          </TableContainer>
+        </Box>
       </MyBox>
       <Flex mt={3} justifyContent={'center'}>
         <Pagination />