diff --git a/public/js/pdf.js b/public/js/pdf.js index 212325e07..94cf1950a 100644 --- a/public/js/pdf.js +++ b/public/js/pdf.js @@ -406,12 +406,12 @@ function getVerbosityLevel() { } function info(msg) { if (verbosity >= VerbosityLevel.INFOS) { - console.log(`Info: ${msg}`); + // console.log(`Info: ${msg}`); } } function warn(msg) { if (verbosity >= VerbosityLevel.WARNINGS) { - console.log(`Warning: ${msg}`); + // console.log(`Warning: ${msg}`); } } function unreachable(msg) { @@ -4206,7 +4206,7 @@ function loadScript(src, removeScriptElement = false) { }); } function deprecated(details) { - console.log("Deprecated API usage: " + details); + // console.log("Deprecated API usage: " + details); } let pdfDateStringRegex; class PDFDateString { diff --git a/public/js/pdf.worker.js b/public/js/pdf.worker.js index d7dc73ac1..b52f5ba80 100644 --- a/public/js/pdf.worker.js +++ b/public/js/pdf.worker.js @@ -1008,12 +1008,12 @@ function getVerbosityLevel() { } function info(msg) { if (verbosity >= VerbosityLevel.INFOS) { - console.log(`Info: ${msg}`); + // console.log(`Info: ${msg}`); } } function warn(msg) { if (verbosity >= VerbosityLevel.WARNINGS) { - console.log(`Warning: ${msg}`); + // console.log(`Warning: ${msg}`); } } function unreachable(msg) { diff --git a/src/pages/api/openapi/chat/chatGpt.ts b/src/pages/api/openapi/chat/chatGpt.ts index 926b01560..7c83b4029 100644 --- a/src/pages/api/openapi/chat/chatGpt.ts +++ b/src/pages/api/openapi/chat/chatGpt.ts @@ -106,10 +106,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) console.log('api response time:', `${(Date.now() - startTime) / 1000}s`); - step = 1; let responseContent = ''; if (isStream) { + step = 1; const streamResponse = await gpt35StreamResponse({ res, stream, diff --git a/src/pages/api/openapi/chat/lafGpt.ts b/src/pages/api/openapi/chat/lafGpt.ts index 230d1a741..c9ad32181 100644 --- a/src/pages/api/openapi/chat/lafGpt.ts +++ b/src/pages/api/openapi/chat/lafGpt.ts @@ -202,10 +202,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) console.log('code response. time:', `${(Date.now() - startTime) / 1000}s`); - step = 1; let responseContent = ''; if (isStream) { + step = 1; const streamResponse = await gpt35StreamResponse({ res, stream, diff --git a/src/pages/api/openapi/chat/vectorGpt.ts b/src/pages/api/openapi/chat/vectorGpt.ts index f8b821d69..39cbca7c2 100644 --- a/src/pages/api/openapi/chat/vectorGpt.ts +++ b/src/pages/api/openapi/chat/vectorGpt.ts @@ -177,10 +177,10 @@ ${ console.log('api response time:', `${(Date.now() - startTime) / 1000}s`); - step = 1; let responseContent = ''; if (isStream) { + step = 1; const streamResponse = await gpt35StreamResponse({ res, stream, diff --git a/src/pages/model/detail/components/SelectFileModal.tsx b/src/pages/model/detail/components/SelectFileModal.tsx index 3e62a29c5..24e10d248 100644 --- a/src/pages/model/detail/components/SelectFileModal.tsx +++ b/src/pages/model/detail/components/SelectFileModal.tsx @@ -62,36 +62,38 @@ const SelectFileModal = ({ const { openConfirm, ConfirmChild } = useConfirm({ content: `确认导入该文件,需要一定时间进行拆解,该任务无法终止!如果余额不足,未完成的任务会被直接清除。一共 ${ splitRes.chunks.length - } 组,大约 ${splitRes.tokens} 个tokens, 约 ${formatPrice( + } 组,大约 ${splitRes.tokens || '数量太多,未计算'} 个tokens, 约 ${formatPrice( splitRes.tokens * modeMap[mode].price )} 元` }); - const fileText = useMemo(() => fileTextArr.join(''), [fileTextArr]); - const onSelectFile = useCallback( async (e: File[]) => { setSelecting(true); try { - const fileTexts = await Promise.all( - e.map((file) => { - // @ts-ignore - const extension = file?.name?.split('.').pop().toLowerCase(); + let promise = Promise.resolve(); + e.map((file) => { + promise = promise.then(async () => { + const extension = file?.name?.split('.')?.pop()?.toLowerCase(); + let text = ''; switch (extension) { case 'txt': case 'md': - return readTxtContent(file); + text = await readTxtContent(file); + break; case 'pdf': - return readPdfContent(file); + text = await readPdfContent(file); + break; case 'doc': case 'docx': - return readDocContent(file); - default: - return ''; + text = await readDocContent(file); + break; } - }) - ); - setFileTextArr(fileTexts); + text && setFileTextArr((state) => [text].concat(state)); + return; + }); + }); + await promise; } catch (error: any) { console.log(error); toast({ @@ -131,6 +133,7 @@ const SelectFileModal = ({ const onclickImport = useCallback(() => { const chunks = fileTextArr + .filter((item) => item) .map((item) => splitText({ text: item, @@ -138,10 +141,15 @@ const SelectFileModal = ({ }) ) .flat(); - // count tokens - const tokens = chunks.map((item) => - countChatTokens({ messages: [{ role: 'system', content: item }] }) - ); + + let tokens: number[] = []; + + // just count 100 sets of tokens + if (chunks.length < 100) { + tokens = chunks.map((item) => + countChatTokens({ messages: [{ role: 'system', content: item }] }) + ); + } setSplitRes({ tokens: tokens.reduce((sum, item) => sum + item, 0), @@ -169,7 +177,7 @@ const SelectFileModal = ({ > 支持 {fileExtension} 文件。模型会自动对文本进行 QA 拆分,需要较长训练时间,拆分需要消耗 - tokens,账号余额不足时,未拆分的数据会被删除。 + tokens,账号余额不足时,未拆分的数据会被删除。一个{fileTextArr.length}个文本。 {/* 拆分模式 */} @@ -200,11 +208,11 @@ const SelectFileModal = ({ )} {/* 文本内容 */} - {fileTextArr.map((item, i) => ( + {fileTextArr.slice(0, 100).map((item, i) => ( 文本{i + 1}