Dataset folder manager (#274)

* feat: retry send

* perf: qa default value

* feat: dataset folder

* feat: kb folder delete and path

* fix: ts

* perf: script load

* feat: fileCard and dataCard

* feat: search file

* feat: max token

* feat: select dataset

* fix: preview chunk

* perf: source update

* export data limit file_id

* docs

* fix: export limit

client/src/utils/plugin/openai.ts

@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
   const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
   const token = adaptMessages.reduce((sum, item) => {
     const text = `${item.role}\n${item.content}`;
-    const enc = getOpenAiEncMap();
-    const encodeText = enc.encode(text);
-    const tokens = encodeText.length + 3; // 补充估算值
+
+    /* use textLen as tokens if encode error */
+    const tokens = (() => {
+      try {
+        const enc = getOpenAiEncMap();
+        const encodeText = enc.encode(text);
+        return encodeText.length + 3; // 补充估算值
+      } catch (error) {
+        return text.length;
+      }
+    })();
+
     return sum + tokens;
   }, 0);
 
@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
 
 export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
   const enc = getOpenAiEncMap();
-  const encodeText = enc.encode(text);
-  const decoder = new TextDecoder();
-  return decoder.decode(enc.decode(encodeText.slice(0, length)));
+
+  try {
+    const encodeText = enc.encode(text);
+    const decoder = new TextDecoder();
+    return decoder.decode(enc.decode(encodeText.slice(0, length)));
+  } catch (error) {
+    return text.slice(0, length);
+  }
 };
 
 export const authOpenAiKey = async (key: string) => {

client/src/utils/sse.ts

@@ -44,7 +44,7 @@ export class SSEParseData {
         data: parseData
       };
     } catch (error) {
-      if (typeof item.data === 'string' && !item.data.startsWith(": ping")) {
+      if (typeof item.data === 'string' && !item.data.startsWith(': ping')) {
         this.storeReadData += item.data;
       } else {
         this.storeReadData = '';

client/src/utils/tools.ts

@@ -59,6 +59,9 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
   return queryParams.toString();
 };
 
+/**
+ * parse string to query object
+ */
 export const parseQueryString = (str: string) => {
   const queryObject: Record<string, any> = {};
 
@@ -125,6 +128,16 @@ export const formatTimeToChatTime = (time: Date) => {
   return target.format('YYYY/M/D');
 };
 
+export const formatFileSize = (bytes: number): string => {
+  if (bytes === 0) return '0 B';
+
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+};
+
 export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
 /**
  * voice broadcast