From 710fa37847f604a746bb727fe73ace78a71e0133 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Thu, 14 Nov 2024 15:50:47 +0800 Subject: [PATCH] fix: chat variable update (#3156) * perf: file encoding * fix: chat variable update --- packages/service/common/file/gridfs/controller.ts | 4 +++- packages/service/common/file/read/utils.ts | 5 +++-- .../service/worker/readFile/extension/rawText.ts | 14 +++++++++++--- .../core/chat/ChatContainer/ChatBox/index.tsx | 6 +++--- projects/app/src/pages/api/common/file/upload.ts | 1 + .../core/dataset/collection/create/localFile.ts | 2 ++ .../dataset/detail/components/Import/Context.tsx | 2 -- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index 541f07fff..2d0d915f6 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -36,6 +36,7 @@ export async function uploadFile({ path, filename, contentType, + encoding, metadata = {} }: { bucketName: `${BucketNameEnum}`; @@ -44,6 +45,7 @@ export async function uploadFile({ path: string; filename: string; contentType?: string; + encoding: string; metadata?: Record; }) { if (!path) return Promise.reject(`filePath is empty`); @@ -52,7 +54,7 @@ export async function uploadFile({ const stats = await fsp.stat(path); if (!stats.isFile()) return Promise.reject(`${path} is not a file`); - const { stream: readStream, encoding } = await stream2Encoding(fs.createReadStream(path)); + const readStream = fs.createReadStream(path); // Add default metadata metadata.teamId = teamId; diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 08a77eacd..578f078b6 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -14,6 +14,7 @@ import { addHours } from 'date-fns'; export type readRawTextByLocalFileParams = { teamId: string; path: string; + encoding: string; metadata?: Record; }; export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => { @@ -22,13 +23,12 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam const extension = path?.split('.')?.pop()?.toLowerCase() || ''; const buffer = fs.readFileSync(path); - const encoding = detectFileEncoding(buffer); const { rawText } = await readRawContentByFileBuffer({ extension, isQAImport: false, teamId: params.teamId, - encoding, + encoding: params.encoding, buffer, metadata: params.metadata }); @@ -53,6 +53,7 @@ export const readRawContentByFileBuffer = async ({ encoding: string; metadata?: Record; }) => { + // Custom read file service const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL; const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || ''; const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false'; diff --git a/packages/service/worker/readFile/extension/rawText.ts b/packages/service/worker/readFile/extension/rawText.ts index d009f5e32..15e0bed83 100644 --- a/packages/service/worker/readFile/extension/rawText.ts +++ b/packages/service/worker/readFile/extension/rawText.ts @@ -18,9 +18,17 @@ const rawEncodingList = [ // 加载源文件内容 export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => { - const content = rawEncodingList.includes(encoding) - ? buffer.toString(encoding as BufferEncoding) - : iconv.decode(buffer, 'gbk'); + const content = (() => { + try { + if (rawEncodingList.includes(encoding)) { + return buffer.toString(encoding as BufferEncoding); + } + + return iconv.decode(buffer, encoding); + } catch (error) { + return buffer.toString('utf-8'); + } + })(); return { rawText: content diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx index 2b12802c0..da6328471 100644 --- a/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx +++ b/projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx @@ -323,7 +323,7 @@ const ChatBox = ( }) }; } else if (event === SseResponseEventEnum.updateVariables && variables) { - variablesForm.reset(variables); + variablesForm.setValue('variables', variables); } else if (event === SseResponseEventEnum.interactive) { const val: AIChatItemValueItemType = { type: ChatItemValueTypeEnum.interactive, @@ -408,7 +408,7 @@ const ChatBox = ( isInteractivePrompt = false }) => { variablesForm.handleSubmit( - async ({ variables }) => { + async ({ variables = {} }) => { if (!onStartChat) return; if (isChatting) { toast({ @@ -435,7 +435,7 @@ const ChatBox = ( // Only declared variables are kept const requestVariables: Record = {}; allVariableList?.forEach((item) => { - requestVariables[item.key] = variables[item.key] || ''; + requestVariables[item.key] = variables[item.key]; }); const responseChatId = getNanoid(24); diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index 7d6c840f3..28c59ec6e 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -51,6 +51,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { path: file.path, filename: file.originalname, contentType: file.mimetype, + encoding: file.encoding, metadata: metadata }); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts b/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts index cee795b04..2d8e964b0 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts @@ -67,6 +67,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse): CreateCo const { rawText } = await readRawTextByLocalFile({ teamId, path: file.path, + encoding: file.encoding, metadata: { ...fileMetadata, relatedId: relatedImgId @@ -81,6 +82,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse): CreateCo path: file.path, filename: file.originalname, contentType: file.mimetype, + encoding: file.encoding, metadata: fileMetadata }); diff --git a/projects/app/src/pages/dataset/detail/components/Import/Context.tsx b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx index 3af357c73..1c78f688f 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/Context.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx @@ -254,8 +254,6 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode icon={} aria-label={''} size={'smSquare'} - w={'26px'} - h={'26px'} borderRadius={'50%'} variant={'whiteBase'} mr={2}