fix: chat variable update (#3156)

* perf: file encoding

* fix: chat variable update
This commit is contained in:
Archer
2024-11-14 15:50:47 +08:00
committed by GitHub
parent e22031ca6c
commit 710fa37847
7 changed files with 23 additions and 11 deletions

View File

@@ -36,6 +36,7 @@ export async function uploadFile({
path, path,
filename, filename,
contentType, contentType,
encoding,
metadata = {} metadata = {}
}: { }: {
bucketName: `${BucketNameEnum}`; bucketName: `${BucketNameEnum}`;
@@ -44,6 +45,7 @@ export async function uploadFile({
path: string; path: string;
filename: string; filename: string;
contentType?: string; contentType?: string;
encoding: string;
metadata?: Record<string, any>; metadata?: Record<string, any>;
}) { }) {
if (!path) return Promise.reject(`filePath is empty`); if (!path) return Promise.reject(`filePath is empty`);
@@ -52,7 +54,7 @@ export async function uploadFile({
const stats = await fsp.stat(path); const stats = await fsp.stat(path);
if (!stats.isFile()) return Promise.reject(`${path} is not a file`); if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
const { stream: readStream, encoding } = await stream2Encoding(fs.createReadStream(path)); const readStream = fs.createReadStream(path);
// Add default metadata // Add default metadata
metadata.teamId = teamId; metadata.teamId = teamId;

View File

@@ -14,6 +14,7 @@ import { addHours } from 'date-fns';
export type readRawTextByLocalFileParams = { export type readRawTextByLocalFileParams = {
teamId: string; teamId: string;
path: string; path: string;
encoding: string;
metadata?: Record<string, any>; metadata?: Record<string, any>;
}; };
export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => { export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
@@ -22,13 +23,12 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
const extension = path?.split('.')?.pop()?.toLowerCase() || ''; const extension = path?.split('.')?.pop()?.toLowerCase() || '';
const buffer = fs.readFileSync(path); const buffer = fs.readFileSync(path);
const encoding = detectFileEncoding(buffer);
const { rawText } = await readRawContentByFileBuffer({ const { rawText } = await readRawContentByFileBuffer({
extension, extension,
isQAImport: false, isQAImport: false,
teamId: params.teamId, teamId: params.teamId,
encoding, encoding: params.encoding,
buffer, buffer,
metadata: params.metadata metadata: params.metadata
}); });
@@ -53,6 +53,7 @@ export const readRawContentByFileBuffer = async ({
encoding: string; encoding: string;
metadata?: Record<string, any>; metadata?: Record<string, any>;
}) => { }) => {
// Custom read file service
const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL; const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL;
const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || ''; const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || '';
const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false'; const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false';

View File

@@ -18,9 +18,17 @@ const rawEncodingList = [
// 加载源文件内容 // 加载源文件内容
export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => { export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
const content = rawEncodingList.includes(encoding) const content = (() => {
? buffer.toString(encoding as BufferEncoding) try {
: iconv.decode(buffer, 'gbk'); if (rawEncodingList.includes(encoding)) {
return buffer.toString(encoding as BufferEncoding);
}
return iconv.decode(buffer, encoding);
} catch (error) {
return buffer.toString('utf-8');
}
})();
return { return {
rawText: content rawText: content

View File

@@ -323,7 +323,7 @@ const ChatBox = (
}) })
}; };
} else if (event === SseResponseEventEnum.updateVariables && variables) { } else if (event === SseResponseEventEnum.updateVariables && variables) {
variablesForm.reset(variables); variablesForm.setValue('variables', variables);
} else if (event === SseResponseEventEnum.interactive) { } else if (event === SseResponseEventEnum.interactive) {
const val: AIChatItemValueItemType = { const val: AIChatItemValueItemType = {
type: ChatItemValueTypeEnum.interactive, type: ChatItemValueTypeEnum.interactive,
@@ -408,7 +408,7 @@ const ChatBox = (
isInteractivePrompt = false isInteractivePrompt = false
}) => { }) => {
variablesForm.handleSubmit( variablesForm.handleSubmit(
async ({ variables }) => { async ({ variables = {} }) => {
if (!onStartChat) return; if (!onStartChat) return;
if (isChatting) { if (isChatting) {
toast({ toast({
@@ -435,7 +435,7 @@ const ChatBox = (
// Only declared variables are kept // Only declared variables are kept
const requestVariables: Record<string, any> = {}; const requestVariables: Record<string, any> = {};
allVariableList?.forEach((item) => { allVariableList?.forEach((item) => {
requestVariables[item.key] = variables[item.key] || ''; requestVariables[item.key] = variables[item.key];
}); });
const responseChatId = getNanoid(24); const responseChatId = getNanoid(24);

View File

@@ -51,6 +51,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
path: file.path, path: file.path,
filename: file.originalname, filename: file.originalname,
contentType: file.mimetype, contentType: file.mimetype,
encoding: file.encoding,
metadata: metadata metadata: metadata
}); });

View File

@@ -67,6 +67,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
const { rawText } = await readRawTextByLocalFile({ const { rawText } = await readRawTextByLocalFile({
teamId, teamId,
path: file.path, path: file.path,
encoding: file.encoding,
metadata: { metadata: {
...fileMetadata, ...fileMetadata,
relatedId: relatedImgId relatedId: relatedImgId
@@ -81,6 +82,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
path: file.path, path: file.path,
filename: file.originalname, filename: file.originalname,
contentType: file.mimetype, contentType: file.mimetype,
encoding: file.encoding,
metadata: fileMetadata metadata: fileMetadata
}); });

View File

@@ -254,8 +254,6 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
icon={<MyIcon name={'common/backFill'} w={'14px'} />} icon={<MyIcon name={'common/backFill'} w={'14px'} />}
aria-label={''} aria-label={''}
size={'smSquare'} size={'smSquare'}
w={'26px'}
h={'26px'}
borderRadius={'50%'} borderRadius={'50%'}
variant={'whiteBase'} variant={'whiteBase'}
mr={2} mr={2}