diff --git a/public/js/pdf.js b/public/js/pdf.js
index 212325e07..94cf1950a 100644
--- a/public/js/pdf.js
+++ b/public/js/pdf.js
@@ -406,12 +406,12 @@ function getVerbosityLevel() {
}
function info(msg) {
if (verbosity >= VerbosityLevel.INFOS) {
- console.log(`Info: ${msg}`);
+ // console.log(`Info: ${msg}`);
}
}
function warn(msg) {
if (verbosity >= VerbosityLevel.WARNINGS) {
- console.log(`Warning: ${msg}`);
+ // console.log(`Warning: ${msg}`);
}
}
function unreachable(msg) {
@@ -4206,7 +4206,7 @@ function loadScript(src, removeScriptElement = false) {
});
}
function deprecated(details) {
- console.log("Deprecated API usage: " + details);
+ // console.log("Deprecated API usage: " + details);
}
let pdfDateStringRegex;
class PDFDateString {
diff --git a/public/js/pdf.worker.js b/public/js/pdf.worker.js
index d7dc73ac1..b52f5ba80 100644
--- a/public/js/pdf.worker.js
+++ b/public/js/pdf.worker.js
@@ -1008,12 +1008,12 @@ function getVerbosityLevel() {
}
function info(msg) {
if (verbosity >= VerbosityLevel.INFOS) {
- console.log(`Info: ${msg}`);
+ // console.log(`Info: ${msg}`);
}
}
function warn(msg) {
if (verbosity >= VerbosityLevel.WARNINGS) {
- console.log(`Warning: ${msg}`);
+ // console.log(`Warning: ${msg}`);
}
}
function unreachable(msg) {
diff --git a/src/pages/api/openapi/chat/chatGpt.ts b/src/pages/api/openapi/chat/chatGpt.ts
index 926b01560..7c83b4029 100644
--- a/src/pages/api/openapi/chat/chatGpt.ts
+++ b/src/pages/api/openapi/chat/chatGpt.ts
@@ -106,10 +106,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
console.log('api response time:', `${(Date.now() - startTime) / 1000}s`);
- step = 1;
let responseContent = '';
if (isStream) {
+ step = 1;
const streamResponse = await gpt35StreamResponse({
res,
stream,
diff --git a/src/pages/api/openapi/chat/lafGpt.ts b/src/pages/api/openapi/chat/lafGpt.ts
index 230d1a741..c9ad32181 100644
--- a/src/pages/api/openapi/chat/lafGpt.ts
+++ b/src/pages/api/openapi/chat/lafGpt.ts
@@ -202,10 +202,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
console.log('code response. time:', `${(Date.now() - startTime) / 1000}s`);
- step = 1;
let responseContent = '';
if (isStream) {
+ step = 1;
const streamResponse = await gpt35StreamResponse({
res,
stream,
diff --git a/src/pages/api/openapi/chat/vectorGpt.ts b/src/pages/api/openapi/chat/vectorGpt.ts
index f8b821d69..39cbca7c2 100644
--- a/src/pages/api/openapi/chat/vectorGpt.ts
+++ b/src/pages/api/openapi/chat/vectorGpt.ts
@@ -177,10 +177,10 @@ ${
console.log('api response time:', `${(Date.now() - startTime) / 1000}s`);
- step = 1;
let responseContent = '';
if (isStream) {
+ step = 1;
const streamResponse = await gpt35StreamResponse({
res,
stream,
diff --git a/src/pages/model/detail/components/SelectFileModal.tsx b/src/pages/model/detail/components/SelectFileModal.tsx
index 3e62a29c5..24e10d248 100644
--- a/src/pages/model/detail/components/SelectFileModal.tsx
+++ b/src/pages/model/detail/components/SelectFileModal.tsx
@@ -62,36 +62,38 @@ const SelectFileModal = ({
const { openConfirm, ConfirmChild } = useConfirm({
content: `确认导入该文件,需要一定时间进行拆解,该任务无法终止!如果余额不足,未完成的任务会被直接清除。一共 ${
splitRes.chunks.length
- } 组,大约 ${splitRes.tokens} 个tokens, 约 ${formatPrice(
+ } 组,大约 ${splitRes.tokens || '数量太多,未计算'} 个tokens, 约 ${formatPrice(
splitRes.tokens * modeMap[mode].price
)} 元`
});
- const fileText = useMemo(() => fileTextArr.join(''), [fileTextArr]);
-
const onSelectFile = useCallback(
async (e: File[]) => {
setSelecting(true);
try {
- const fileTexts = await Promise.all(
- e.map((file) => {
- // @ts-ignore
- const extension = file?.name?.split('.').pop().toLowerCase();
+ let promise = Promise.resolve();
+ e.map((file) => {
+ promise = promise.then(async () => {
+ const extension = file?.name?.split('.')?.pop()?.toLowerCase();
+ let text = '';
switch (extension) {
case 'txt':
case 'md':
- return readTxtContent(file);
+ text = await readTxtContent(file);
+ break;
case 'pdf':
- return readPdfContent(file);
+ text = await readPdfContent(file);
+ break;
case 'doc':
case 'docx':
- return readDocContent(file);
- default:
- return '';
+ text = await readDocContent(file);
+ break;
}
- })
- );
- setFileTextArr(fileTexts);
+ text && setFileTextArr((state) => [text].concat(state));
+ return;
+ });
+ });
+ await promise;
} catch (error: any) {
console.log(error);
toast({
@@ -131,6 +133,7 @@ const SelectFileModal = ({
const onclickImport = useCallback(() => {
const chunks = fileTextArr
+ .filter((item) => item)
.map((item) =>
splitText({
text: item,
@@ -138,10 +141,15 @@ const SelectFileModal = ({
})
)
.flat();
- // count tokens
- const tokens = chunks.map((item) =>
- countChatTokens({ messages: [{ role: 'system', content: item }] })
- );
+
+ let tokens: number[] = [];
+
+ // just count 100 sets of tokens
+ if (chunks.length < 100) {
+ tokens = chunks.map((item) =>
+ countChatTokens({ messages: [{ role: 'system', content: item }] })
+ );
+ }
setSplitRes({
tokens: tokens.reduce((sum, item) => sum + item, 0),
@@ -169,7 +177,7 @@ const SelectFileModal = ({
>
支持 {fileExtension} 文件。模型会自动对文本进行 QA 拆分,需要较长训练时间,拆分需要消耗
- tokens,账号余额不足时,未拆分的数据会被删除。
+ tokens,账号余额不足时,未拆分的数据会被删除。一个{fileTextArr.length}个文本。
{/* 拆分模式 */}
@@ -200,11 +208,11 @@ const SelectFileModal = ({
)}
{/* 文本内容 */}
- {fileTextArr.map((item, i) => (
+ {fileTextArr.slice(0, 100).map((item, i) => (
文本{i + 1}