diff --git a/client/src/constants/common.ts b/client/src/constants/common.ts index acc2c87bb..2174abcad 100644 --- a/client/src/constants/common.ts +++ b/client/src/constants/common.ts @@ -6,11 +6,11 @@ export enum UserAuthTypeEnum { export const PRICE_SCALE = 100000; export const fileImgs = [ - { reg: /pdf/gi, src: '/imgs/files/pdf.svg' }, - { reg: /csv/gi, src: '/imgs/files/csv.svg' }, - { reg: /(doc|docs)/gi, src: '/imgs/files/doc.svg' }, - { reg: /txt/gi, src: '/imgs/files/txt.svg' }, - { reg: /md/gi, src: '/imgs/files/markdown.svg' } + { suffix: 'pdf', src: '/imgs/files/pdf.svg' }, + { suffix: 'csv', src: '/imgs/files/csv.svg' }, + { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' }, + { suffix: 'txt', src: '/imgs/files/txt.svg' }, + { suffix: 'md', src: '/imgs/files/markdown.svg' } ]; export const htmlTemplate = ` diff --git a/client/src/pages/kb/detail/components/Import/FileSelect.tsx b/client/src/pages/kb/detail/components/Import/FileSelect.tsx index 791308017..751d5813e 100644 --- a/client/src/pages/kb/detail/components/Import/FileSelect.tsx +++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx @@ -76,76 +76,76 @@ const FileSelect = ({ setSelecting(true); try { // Parse file by file - let promise = Promise.resolve([]); - files.forEach((file) => { - promise = promise.then(async (result) => { - const extension = file?.name?.split('.')?.pop()?.toLowerCase(); + const chunkFiles: FileItemType[] = []; - /* text file */ - const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src; - let text = await (async () => { - switch (extension) { - case 'txt': - case 'md': - return readTxtContent(file); - case 'pdf': - return readPdfContent(file); - case 'doc': - case 'docx': - return readDocContent(file); - } - return ''; - })(); + for await (let file of files) { + const extension = file?.name?.split('.')?.pop()?.toLowerCase(); - if (!icon) return result; + /* text file */ + const icon = fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.name))?.src; - if (text) { - text = simpleText(text); - const splitRes = splitText2Chunks({ - text, - maxLen: chunkLen - }); - const fileItem: FileItemType = { - id: nanoid(), - filename: file.name, - icon, - text, - tokens: splitRes.tokens, - chunks: splitRes.chunks.map((chunk) => ({ - q: chunk, - a: '', - source: file.name - })) - }; - return [fileItem].concat(result); + if (!icon) { + continue; + } + + let text = await (async () => { + switch (extension) { + case 'txt': + case 'md': + return readTxtContent(file); + case 'pdf': + return readPdfContent(file); + case 'doc': + case 'docx': + return readDocContent(file); } + return ''; + })(); - /* csv file */ - if (extension === 'csv') { - const { header, data } = await readCsvContent(file); - if (header[0] !== 'question' || header[1] !== 'answer') { - throw new Error('csv 文件格式有误,请确保 question 和 answer 两列'); - } - const fileItem: FileItemType = { - id: nanoid(), - filename: file.name, - icon, - tokens: 0, - text: '', - chunks: data.map((item) => ({ - q: item[0], - a: item[1], - source: item[2] || file.name - })) - }; - return [fileItem].concat(result); + if (text) { + text = simpleText(text); + const splitRes = splitText2Chunks({ + text, + maxLen: chunkLen + }); + const fileItem: FileItemType = { + id: nanoid(), + filename: file.name, + icon, + text, + tokens: splitRes.tokens, + chunks: splitRes.chunks.map((chunk) => ({ + q: chunk, + a: '', + source: file.name + })) + }; + chunkFiles.unshift(fileItem); + continue; + } + + /* csv file */ + if (extension === 'csv') { + const { header, data } = await readCsvContent(file); + if (header[0] !== 'question' || header[1] !== 'answer') { + throw new Error('csv 文件格式有误,请确保 question 和 answer 两列'); } - return result; - }); - }); - - const chunkFiles = await promise; + const fileItem: FileItemType = { + id: nanoid(), + filename: file.name, + icon, + tokens: 0, + text: '', + chunks: data.map((item) => ({ + q: item[0], + a: item[1], + source: item[2] || file.name + })) + }; + chunkFiles.unshift(fileItem); + } + } onPushFiles(chunkFiles); } catch (error: any) { console.log(error); diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts index 2e3f7a902..2c814c12a 100644 --- a/client/src/service/events/generateQA.ts +++ b/client/src/service/events/generateQA.ts @@ -9,6 +9,7 @@ import { axiosConfig, getAIChatApi } from '../ai/openai'; import { ChatCompletionRequestMessage } from 'openai'; import { modelToolMap } from '@/utils/plugin'; import { gptMessage2ChatType } from '@/utils/adapt'; +import { addLog } from '../utils/tools'; const reduceQueue = () => { global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0; @@ -105,12 +106,16 @@ A2: const result = formatSplitText(answer || ''); // 格式化后的QA对 console.log(`split result length: `, result.length); // 计费 - result.length > 0 && + if (result.length > 0) { pushQABill({ userId: data.userId, totalTokens, appName: 'QA 拆分' }); + } else { + addLog.info(`QA result 0:`, { answer }); + } + return { rawContent: answer, result