4.6.7 first pr (#726)

packages/global/common/string/markdown.ts

@@ -15,10 +15,10 @@ export const simpleMarkdownText = (rawText: string) => {
     return `[${cleanedLinkText}](${url})`;
   });
 
-  // replace special \.* ……
-  const reg1 = /\\([-.!`_(){}\[\]])/g;
+  // replace special #\.* ……
+  const reg1 = /\\([#`!*()+-_\[\]{}\\.])/g;
   if (reg1.test(rawText)) {
-    rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
+    rawText = rawText.replace(reg1, '$1');
   }
 
   // replace \\n
@@ -45,24 +45,26 @@ export const uploadMarkdownBase64 = async ({
   uploadImgController
 }: {
   rawText: string;
-  uploadImgController: (base64: string) => Promise<string>;
+  uploadImgController?: (base64: string) => Promise<string>;
 }) => {
-  // match base64, upload and replace it
-  const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
-  const base64Arr = rawText.match(base64Regex) || [];
-  // upload base64 and replace it
-  await Promise.all(
-    base64Arr.map(async (base64Img) => {
-      try {
-        const str = await uploadImgController(base64Img);
+  if (uploadImgController) {
+    // match base64, upload and replace it
+    const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
+    const base64Arr = rawText.match(base64Regex) || [];
+    // upload base64 and replace it
+    await Promise.all(
+      base64Arr.map(async (base64Img) => {
+        try {
+          const str = await uploadImgController(base64Img);
 
-        rawText = rawText.replace(base64Img, str);
-      } catch (error) {
-        rawText = rawText.replace(base64Img, '');
-        rawText = rawText.replace(/!\[.*\]\(\)/g, '');
-      }
-    })
-  );
+          rawText = rawText.replace(base64Img, str);
+        } catch (error) {
+          rawText = rawText.replace(base64Img, '');
+          rawText = rawText.replace(/!\[.*\]\(\)/g, '');
+        }
+      })
+    );
+  }
 
   // Remove white space on both sides of the picture
   const trimReg = /(!\[.*\]\(.*\))\s*/g;
@@ -70,5 +72,20 @@ export const uploadMarkdownBase64 = async ({
     rawText = rawText.replace(trimReg, '$1');
   }
 
-  return simpleMarkdownText(rawText);
+  return rawText;
+};
+
+export const markdownProcess = async ({
+  rawText,
+  uploadImgController
+}: {
+  rawText: string;
+  uploadImgController?: (base64: string) => Promise<string>;
+}) => {
+  const imageProcess = await uploadMarkdownBase64({
+    rawText,
+    uploadImgController
+  });
+
+  return simpleMarkdownText(imageProcess);
 };

packages/global/common/string/tiktoken/index.ts

@@ -33,6 +33,12 @@ export function countPromptTokens(
 ) {
   const enc = getTikTokenEnc();
   const text = `${role}\n${prompt}`;
+
+  // too large a text will block the thread
+  if (text.length > 15000) {
+    return text.length * 1.7;
+  }
+
   try {
     const encodeText = enc.encode(text);
     return encodeText.length + role.length; // 补充 role 估算值

packages/global/common/string/time.ts

@@ -1,3 +1,4 @@
 import dayjs from 'dayjs';
 
-export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm');
+export const formatTime2YMDHM = (time?: Date) =>
+  time ? dayjs(time).format('YYYY-MM-DD HH:mm') : '';

packages/global/common/string/tools.ts

@@ -1,4 +1,5 @@
 import crypto from 'crypto';
+import { customAlphabet } from 'nanoid';
 
 /* check string is a web link */
 export function strIsLink(str?: string) {
@@ -36,3 +37,7 @@ export function replaceVariable(text: string, obj: Record<string, string | numbe
   }
   return text || '';
 }
+
+export const getNanoid = (size = 12) => {
+  return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)();
+};