remove markdown format;refresh username;perf: latext render (#3877)

* refresh username * remove md format * perf: latext render * ignore big image * model config
2025-07-21 03:35:36 +00:00 · 2025-02-25 16:16:30 +08:00
parent 4f0dd96699
commit 5d5bee9e41
9 changed files with 108 additions and 130 deletions
--- a/docSite/content/zh-cn/docs/development/upgrading/4823.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/4823.md
@@ -15,8 +15,11 @@ weight: 802
 ## ⚙️ 优化

 1. 模型配置表单，增加必填项校验。
-2. 集合列表数据统计方式。
+2. 集合列表数据统计方式，提高大数据量统计性能。
+3. 优化数学公式，转义 Latex 格式成 Markdown 格式。
+4. 解析文档图片，图片太大时，自动忽略。

 ## 🐛 修复

-1. 标签过滤时，子文件夹未成功过滤。
+1. 标签过滤时，子文件夹未成功过滤。
+2. 暂时移除 md 阅读优化，避免链接分割错误。
--- a/packages/service/common/file/read/utils.ts
+++ b/packages/service/common/file/read/utils.ts
@@ -111,15 +111,21 @@ export const readRawContentByFileBuffer = async ({
  // markdown data format
  if (imageList) {
    await batchRun(imageList, async (item) => {
-      const src = await uploadMongoImg({
-        base64Img: `data:${item.mime};base64,${item.base64}`,
-        teamId,
-        // expiredTime: addHours(new Date(), 1),
-        metadata: {
-          ...metadata,
-          mime: item.mime
+      const src = await (async () => {
+        try {
+          return await uploadMongoImg({
+            base64Img: `data:${item.mime};base64,${item.base64}`,
+            teamId,
+            // expiredTime: addHours(new Date(), 1),
+            metadata: {
+              ...metadata,
+              mime: item.mime
+            }
+          });
+        } catch (error) {
+          return '';
        }
-      });
+      })();
      rawText = rawText.replace(item.uuid, src);
      if (formatText) {
        formatText = formatText.replace(item.uuid, src);
--- a/packages/service/core/ai/config/provider/Claude.json
+++ b/packages/service/core/ai/config/provider/Claude.json
@@ -1,6 +1,30 @@
 {
  "provider": "Claude",
  "list": [
+    {
+      "model": "claude-3-7-sonnet-20250219",
+      "name": "claude-3-7-sonnet-20250219",
+      "maxContext": 200000,
+      "maxResponse": 8000,
+      "quoteMaxToken": 100000,
+      "maxTemperature": 1,
+      "showTopP": true,
+      "showStopSign": true,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm"
+    },
    {
      "model": "claude-3-5-haiku-20241022",
      "name": "claude-3-5-haiku-20241022",
@@ -10,7 +34,7 @@
      "maxTemperature": 1,
      "showTopP": true,
      "showStopSign": true,
-      "vision": false,
+      "vision": true,
      "toolChoice": true,
      "functionCall": false,
      "defaultSystemChatPrompt": "",
@@ -98,4 +122,4 @@
      "type": "llm"
    }
  ]
-}
+}
--- a/packages/service/core/ai/config/provider/Grok.json
+++ b/packages/service/core/ai/config/provider/Grok.json
@@ -1,4 +1,29 @@
 {
  "provider": "Grok",
-  "list": []
-}
+  "list": [
+    {
+      "model": "grok-3",
+      "name": "grok-3",
+      "maxContext": 128000,
+      "maxResponse": 8000,
+      "quoteMaxToken": 128000,
+      "maxTemperature": 1,
+      "showTopP": true,
+      "showStopSign": true,
+      "vision": false,
+      "toolChoice": false,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm"
+    }
+  ]
+}
--- a/projects/app/next.config.js
+++ b/projects/app/next.config.js
@@ -105,22 +105,6 @@ function getWorkerConfig() {
      .isDirectory();
  });

-  /* 
-    {
-      'worker/htmlStr2Md': path.resolve(
-                process.cwd(),
-                '../../packages/service/worker/htmlStr2Md/index.ts'
-              ),
-              'worker/countGptMessagesTokens': path.resolve(
-                process.cwd(),
-                '../../packages/service/worker/countGptMessagesTokens/index.ts'
-              ),
-              'worker/readFile': path.resolve(
-                process.cwd(),
-                '../../packages/service/worker/readFile/index.ts'
-              )
-    }
-  */
  const workerConfig = folderList.reduce((acc, item) => {
    acc[`worker/${item}`] = path.resolve(
      process.cwd(),
--- a/projects/app/src/components/Layout/index.tsx
+++ b/projects/app/src/components/Layout/index.tsx
@@ -52,7 +52,6 @@ export const navbarWidth = '64px';
 const Layout = ({ children }: { children: JSX.Element }) => {
  const router = useRouter();
  const { t } = useTranslation();
-  const { toast } = useToast();
  const { Loading } = useLoading();
  const { loading, feConfigs, notSufficientModalType, llmModelList, embeddingModelList } =
    useSystemStore();
@@ -88,6 +87,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
  });

  // Check model invalid
+  const { toast } = useToast();
  useDebounceEffect(
    () => {
      if (userInfo?.username === 'root') {
@@ -96,13 +96,13 @@ const Layout = ({ children }: { children: JSX.Element }) => {
            status: 'warning',
            title: t('common:llm_model_not_config')
          });
-          router.push('/account/model');
+          router.pathname !== '/account/model' && router.push('/account/model');
        } else if (embeddingModelList.length === 0) {
          toast({
            status: 'warning',
            title: t('common:embedding_model_not_config')
          });
-          router.push('/account/model');
+          router.pathname !== '/account/model' && router.push('/account/model');
        }
      }
    },
--- a/projects/app/src/components/Markdown/index.tsx
+++ b/projects/app/src/components/Markdown/index.tsx
@@ -11,7 +11,7 @@ import styles from './index.module.scss';
 import dynamic from 'next/dynamic';

 import { Box } from '@chakra-ui/react';
-import { CodeClassNameEnum } from './utils';
+import { CodeClassNameEnum, mdTextFormat } from './utils';

 const CodeLight = dynamic(() => import('./codeBlock/CodeLight'), { ssr: false });
 const MermaidCodeBlock = dynamic(() => import('./img/MermaidCodeBlock'), { ssr: false });
@@ -54,36 +54,7 @@ const MarkdownRender = ({ source = '', showAnimation, isDisabled, forbidZhFormat

  const formatSource = useMemo(() => {
    if (showAnimation || forbidZhFormat) return source;
-
-    // 保护 URL 格式：https://, http://, /api/xxx
-    const urlPlaceholders: string[] = [];
-    const textWithProtectedUrls = source.replace(
-      /https?:\/\/(?:(?:[\w-]+\.)+[a-zA-Z]{2,6}|localhost)(?::\d{2,5})?(?:\/[\w\-./?%&=@]*)?/g,
-      (match) => {
-        urlPlaceholders.push(match);
-        return `__URL_${urlPlaceholders.length - 1}__ `;
-      }
-    );
-
-    // 处理中文与英文数字之间的分词
-    const textWithSpaces = textWithProtectedUrls
-      .replace(
-        /([\u4e00-\u9fa5\u3000-\u303f])([a-zA-Z0-9])|([a-zA-Z0-9])([\u4e00-\u9fa5\u3000-\u303f])/g,
-        '$1$3 $2$4'
-      )
-      // 处理引用标记
-      .replace(/\n*(\[QUOTE SIGN\]\(.*\))/g, '$1')
-      // 处理 [quote:id] 格式引用，将 [quote:675934a198f46329dfc6d05a] 转换为 [675934a198f46329dfc6d05a](QUOTE)
-      .replace(/\[quote:?\s*([a-f0-9]{24})\](?!\()/gi, '[$1](QUOTE)')
-      .replace(/\[([a-f0-9]{24})\](?!\()/g, '[$1](QUOTE)');
-
-    // 还原 URL
-    const finalText = textWithSpaces.replace(
-      /__URL_(\d+)__/g,
-      (_, index) => `${urlPlaceholders[parseInt(index)]}`
-    );
-
-    return finalText;
+    return mdTextFormat(source);
  }, [forbidZhFormat, showAnimation, source]);

  const urlTransform = useCallback((val: string) => {
--- a/projects/app/src/components/Markdown/utils.ts
+++ b/projects/app/src/components/Markdown/utils.ts
@@ -12,70 +12,34 @@ export enum CodeClassNameEnum {
  audio = 'audio'
 }

-function htmlTableToLatex(html: string) {
-  const parser = new DOMParser();
-  const doc = parser.parseFromString(html, 'text/html');
-  const table = doc.querySelector('table');
-
-  if (!table) return '';
-
-  let latex = '\\begin{tabular}{';
-
-  // 获取列数
-  const columns = table.querySelectorAll('tr:first-child th, tr:first-child td').length;
-  latex += '|' + 'c|'.repeat(columns) + '}\n\\hline\n';
-
-  // 创建一个二维数组来跟踪单元格合并情况
-  const cellTracker = Array.from({ length: table.rows.length }, () => Array(columns).fill(false));
-
-  // 遍历行
-  table.querySelectorAll('tr').forEach((row, rowIndex) => {
-    const cells = row.querySelectorAll('th, td');
-    let cellTexts: string[] = [];
-    let colIndex = 0;
-
-    cells.forEach((cell) => {
-      // 跳过已经被合并的单元格
-      while (cellTracker[rowIndex][colIndex]) {
-        colIndex++;
+export const mdTextFormat = (text: string) => {
+  // NextChat function - Format latex to $$
+  const escapeBrackets = (text: string) => {
+    const pattern = /(```[\s\S]*?```|`.*?`)|\\\[([\s\S]*?[^\\])\\\]|\\\((.*?)\\\)/g;
+    return text.replace(pattern, (match, codeBlock, squareBracket, roundBracket) => {
+      if (codeBlock) {
+        return codeBlock;
+      } else if (squareBracket) {
+        return `$$${squareBracket}$$`;
+      } else if (roundBracket) {
+        return `$${roundBracket}$`;
      }
-
-      // @ts-ignore
-      const rowspan = parseInt(cell.getAttribute('rowspan') || 1, 10);
-      // @ts-ignore
-      const colspan = parseInt(cell.getAttribute('colspan') || 1, 10);
-
-      // 添加单元格内容
-      let cellText = cell.textContent?.trim() || '';
-      if (colspan > 1) {
-        cellText = `\\multicolumn{${colspan}}{|c|}{${cellText}}`;
-      }
-      if (rowspan > 1) {
-        cellText = `\\multirow{${rowspan}}{*}{${cellText}}`;
-      }
-      cellTexts.push(cellText);
-
-      // 标记合并的单元格
-      for (let i = 0; i < rowspan; i++) {
-        for (let j = 0; j < colspan; j++) {
-          cellTracker[rowIndex + i][colIndex + j] = true;
-        }
-      }
-
-      colIndex += colspan;
+      return match;
    });
+  };
+  // 处理 [quote:id] 格式引用，将 [quote:675934a198f46329dfc6d05a] 转换为 [675934a198f46329dfc6d05a](QUOTE)
+  const formatQuote = (text: string) => {
+    return (
+      text
+        // .replace(
+        //   /([\u4e00-\u9fa5\u3000-\u303f])([a-zA-Z0-9])|([a-zA-Z0-9])([\u4e00-\u9fa5\u3000-\u303f])/g,
+        //   '$1$3 $2$4'
+        // )
+        // 处理 [quote:id] 格式引用，将 [quote:675934a198f46329dfc6d05a] 转换为 [675934a198f46329dfc6d05a](QUOTE)
+        .replace(/\[quote:?\s*([a-f0-9]{24})\](?!\()/gi, '[$1](QUOTE)')
+        .replace(/\[([a-f0-9]{24})\](?!\()/g, '[$1](QUOTE)')
+    );
+  };

-    latex += cellTexts.join(' & ') + ' \\\\\n\\hline\n';
-  });
-
-  latex += '\\end{tabular}';
-
-  return `\`\`\`${CodeClassNameEnum.latex}
-    ${latex}
-    \`\`\``;
-}
-
-export function convertHtmlTablesToLatex(input: string) {
-  const tableRegex = /<table[\s\S]*?<\/table>/gi;
-  return input.replace(tableRegex, (match) => htmlTableToLatex(match));
-}
+  return formatQuote(escapeBrackets(text));
+};
--- a/projects/app/src/pages/account/info/index.tsx
+++ b/projects/app/src/pages/account/info/index.tsx
@@ -234,11 +234,12 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
              borderColor={'transparent'}
              transform={'translateX(-11px)'}
              maxLength={20}
-              onBlur={(e) => {
+              onBlur={async (e) => {
                const val = e.target.value;
                if (val === userInfo?.team?.memberName) return;
                try {
-                  putUpdateMemberName(val);
+                  await putUpdateMemberName(val);
+                  initUserInfo();
                } catch (error) {}
              }}
            />