perf: csv文件选择

src/utils/tools.ts

@@ -1,6 +1,5 @@
 import crypto from 'crypto';
 import { useToast } from '@/hooks/useToast';
-import mammoth from 'mammoth';
 
 /**
  * copy text data
@@ -34,11 +33,17 @@ export const useCopyData = () => {
   };
 };
 
+/**
+ * 密码加密
+ */
 export const createHashPassword = (text: string) => {
   const hash = crypto.createHash('sha256').update(text).digest('hex');
   return hash;
 };
 
+/**
+ * 对象转成 query 字符串
+ */
 export const Obj2Query = (obj: Record<string, string | number>) => {
   const queryParams = new URLSearchParams();
   for (const key in obj) {
@@ -47,86 +52,6 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
   return queryParams.toString();
 };
 
-/**
- * 读取 txt 文件内容
- */
-export const readTxtContent = (file: File) => {
-  return new Promise((resolve: (_: string) => void, reject) => {
-    const reader = new FileReader();
-    reader.onload = () => {
-      resolve(reader.result as string);
-    };
-    reader.onerror = (err) => {
-      console.log('error txt read:', err);
-      reject('读取 txt 文件失败');
-    };
-    reader.readAsText(file);
-  });
-};
-
-/**
- * 读取 pdf 内容
- */
-export const readPdfContent = (file: File) =>
-  new Promise<string>((resolve, reject) => {
-    const pdfjsLib = window['pdfjs-dist/build/pdf'];
-    pdfjsLib.workerSrc = '/js/pdf.worker.js';
-
-    const readPDFPage = async (doc: any, pageNo: number) => {
-      const page = await doc.getPage(pageNo);
-      const tokenizedText = await page.getTextContent();
-      const pageText = tokenizedText.items.map((token: any) => token.str).join(' ');
-      return pageText;
-    };
-
-    let reader = new FileReader();
-    reader.readAsArrayBuffer(file);
-    reader.onload = async (event) => {
-      if (!event?.target?.result) return reject('解析 PDF 失败');
-      try {
-        const doc = await pdfjsLib.getDocument(event.target.result).promise;
-        const pageTextPromises = [];
-        for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
-          pageTextPromises.push(readPDFPage(doc, pageNo));
-        }
-        const pageTexts = await Promise.all(pageTextPromises);
-        resolve(pageTexts.join('\n'));
-      } catch (err) {
-        console.log(err, 'pdfjs error');
-        reject('解析 PDF 失败');
-      }
-    };
-    reader.onerror = (err) => {
-      console.log(err, 'reader error');
-      reject('解析 PDF 失败');
-    };
-  });
-
-/**
- * 读取doc
- */
-export const readDocContent = (file: File) =>
-  new Promise<string>((resolve, reject) => {
-    const reader = new FileReader();
-    reader.readAsArrayBuffer(file);
-    reader.onload = async ({ target }) => {
-      if (!target?.result) return reject('读取 doc 文件失败');
-      try {
-        const res = await mammoth.extractRawText({
-          arrayBuffer: target.result as ArrayBuffer
-        });
-        resolve(res?.value);
-      } catch (error) {
-        reject('读取 doc 文件失败, 请转换成 PDF');
-      }
-    };
-    reader.onerror = (err) => {
-      console.log('error doc read:', err);
-
-      reject('读取 doc 文件失败');
-    };
-  });
-
 /**
  * 向量转成 float32 buffer 格式
  */
@@ -138,11 +63,18 @@ export const vectorToBuffer = (vector: number[]) => {
   return buffer;
 };
 
-export function formatVector(vector: number[]) {
+export const formatVector = (vector: number[]) => {
   let formattedVector = vector.slice(0, 1536); // 截取前1536个元素
   if (vector.length > 1536) {
     formattedVector = formattedVector.concat(Array(1536 - formattedVector.length).fill(0)); // 在后面添加0
   }
 
   return formattedVector;
-}
+};
+
+/**
+ * 字符串清理，替换换行符号
+ */
+export const clearStrLineBreak = (str: string) => {
+  return str.replace(/\n/g, '\n').replace(/\n/g, '\\n').trim();
+};