mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-24 13:53:50 +00:00
4.8.11 test fix (#2746)
* fix: refresh tool param * perf: load image message * fix: cannot remoce externalReadUrl * perf: variables update dom * perf: empty response tip * fix: conflict
This commit is contained in:
@@ -48,7 +48,7 @@ export const Prompt_userQuotePromptList: PromptTemplateItem[] = [
|
||||
{
|
||||
title: i18nT('app:template.standard_template'),
|
||||
desc: '',
|
||||
value: `使用 <Reference></Reference> 标记中的内容作为本次对话的参考内容:
|
||||
value: `使用 <Reference></Reference> 标记中的内容作为本次对话的参考:
|
||||
|
||||
<Reference>
|
||||
{{quote}}
|
||||
@@ -83,7 +83,7 @@ export const Prompt_userQuotePromptList: PromptTemplateItem[] = [
|
||||
{
|
||||
title: i18nT('app:template.standard_strict'),
|
||||
desc: '',
|
||||
value: `忘记你已有的知识,仅使用 <Reference></Reference> 标记中的内容作为本次对话的参考内容:
|
||||
value: `忘记你已有的知识,仅使用 <Reference></Reference> 标记中的内容作为本次对话的参考:
|
||||
|
||||
<Reference>
|
||||
{{quote}}
|
||||
@@ -133,7 +133,7 @@ export const Prompt_systemQuotePromptList: PromptTemplateItem[] = [
|
||||
{
|
||||
title: i18nT('app:template.standard_template'),
|
||||
desc: '',
|
||||
value: `使用 <Reference></Reference> 标记中的内容作为本次对话的参考内容:
|
||||
value: `使用 <Reference></Reference> 标记中的内容作为本次对话的参考:
|
||||
|
||||
<Reference>
|
||||
{{quote}}
|
||||
@@ -164,7 +164,7 @@ export const Prompt_systemQuotePromptList: PromptTemplateItem[] = [
|
||||
{
|
||||
title: i18nT('app:template.standard_strict'),
|
||||
desc: '',
|
||||
value: `忘记你已有的知识,仅使用 <Reference></Reference> 标记中的内容作为本次对话的参考内容:
|
||||
value: `忘记你已有的知识,仅使用 <Reference></Reference> 标记中的内容作为本次对话的参考:
|
||||
|
||||
<Reference>
|
||||
{{quote}}
|
||||
@@ -207,7 +207,7 @@ export const Prompt_systemQuotePromptList: PromptTemplateItem[] = [
|
||||
];
|
||||
|
||||
// Document quote prompt
|
||||
export const Prompt_DocumentQuote = `将 <Reference></Reference> 中的内容作为本次对话的参考内容:
|
||||
export const Prompt_DocumentQuote = `将 <Reference></Reference> 中的内容作为本次对话的参考:
|
||||
<Reference>
|
||||
{{quote}}
|
||||
</Reference>
|
||||
|
@@ -8,6 +8,8 @@ import axios from 'axios';
|
||||
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
|
||||
import { getFileContentTypeFromHeader, guessBase64ImageType } from '../../common/file/utils';
|
||||
import { serverRequestBaseUrl } from '../../common/api/serverRequest';
|
||||
import { i18nT } from '../../../web/i18n/utils';
|
||||
import { addLog } from '../../common/system/log';
|
||||
|
||||
/* slice chat context by tokens */
|
||||
const filterEmptyMessages = (messages: ChatCompletionMessageParam[]) => {
|
||||
@@ -111,20 +113,62 @@ export const loadRequestMessages = async ({
|
||||
useVision?: boolean;
|
||||
origin?: string;
|
||||
}) => {
|
||||
// Load image to base64
|
||||
const loadImageToBase64 = async (messages: ChatCompletionContentPart[]) => {
|
||||
return Promise.all(
|
||||
messages.map(async (item) => {
|
||||
if (item.type === 'image_url') {
|
||||
// Remove url origin
|
||||
const imgUrl = (() => {
|
||||
if (origin && item.image_url.url.startsWith(origin)) {
|
||||
return item.image_url.url.replace(origin, '');
|
||||
}
|
||||
return item.image_url.url;
|
||||
})();
|
||||
|
||||
// If imgUrl is a local path, load image from local, and set url to base64
|
||||
if (imgUrl.startsWith('/') || process.env.VISION_FOCUS_BASE64 === 'true') {
|
||||
addLog.debug('Load image from local server', {
|
||||
baseUrl: serverRequestBaseUrl,
|
||||
requestUrl: imgUrl
|
||||
});
|
||||
const response = await axios.get(imgUrl, {
|
||||
baseURL: serverRequestBaseUrl,
|
||||
responseType: 'arraybuffer',
|
||||
proxy: false
|
||||
});
|
||||
const base64 = Buffer.from(response.data, 'binary').toString('base64');
|
||||
const imageType =
|
||||
getFileContentTypeFromHeader(response.headers['content-type']) ||
|
||||
guessBase64ImageType(base64);
|
||||
|
||||
return {
|
||||
...item,
|
||||
image_url: {
|
||||
...item.image_url,
|
||||
url: `data:${imageType};base64,${base64}`
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
return item;
|
||||
})
|
||||
);
|
||||
};
|
||||
// Split question text and image
|
||||
function parseStringWithImages(input: string): ChatCompletionContentPart[] {
|
||||
const parseStringWithImages = (input: string): ChatCompletionContentPart[] => {
|
||||
if (!useVision) {
|
||||
return [{ type: 'text', text: input || '' }];
|
||||
}
|
||||
|
||||
// 正则表达式匹配图片URL
|
||||
const imageRegex =
|
||||
/(https?:\/\/[^\s/$.?#].[^\s]*\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg|ico|heic|avif))/i;
|
||||
/(https?:\/\/[^\s/$.?#].[^\s]*\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg|ico|heic|avif))/gi;
|
||||
|
||||
const result: ChatCompletionContentPart[] = [];
|
||||
|
||||
// 提取所有HTTPS图片URL并添加到result开头
|
||||
const httpsImages = input.match(imageRegex) || [];
|
||||
const httpsImages = [...new Set(Array.from(input.matchAll(imageRegex), (m) => m[0]))];
|
||||
httpsImages.forEach((url) => {
|
||||
result.push({
|
||||
type: 'image_url',
|
||||
@@ -137,54 +181,27 @@ export const loadRequestMessages = async ({
|
||||
// 添加原始input作为文本
|
||||
result.push({ type: 'text', text: input });
|
||||
return result;
|
||||
}
|
||||
// Load image
|
||||
};
|
||||
// Parse user content(text and img)
|
||||
const parseUserContent = async (content: string | ChatCompletionContentPart[]) => {
|
||||
if (typeof content === 'string') {
|
||||
return parseStringWithImages(content);
|
||||
return loadImageToBase64(parseStringWithImages(content));
|
||||
}
|
||||
|
||||
const result = await Promise.all(
|
||||
content.map(async (item) => {
|
||||
if (item.type === 'text') return parseStringWithImages(item.text);
|
||||
if (item.type === 'file_url') return;
|
||||
if (item.type === 'file_url') return; // LLM not support file_url
|
||||
|
||||
if (!item.image_url.url) return item;
|
||||
|
||||
// Remove url origin
|
||||
const imgUrl = (() => {
|
||||
if (origin && item.image_url.url.startsWith(origin)) {
|
||||
return item.image_url.url.replace(origin, '');
|
||||
}
|
||||
return item.image_url.url;
|
||||
})();
|
||||
|
||||
/* Load local image */
|
||||
if (imgUrl.startsWith('/')) {
|
||||
const response = await axios.get(imgUrl, {
|
||||
baseURL: serverRequestBaseUrl,
|
||||
responseType: 'arraybuffer'
|
||||
});
|
||||
const base64 = Buffer.from(response.data, 'binary').toString('base64');
|
||||
const imageType =
|
||||
getFileContentTypeFromHeader(response.headers['content-type']) ||
|
||||
guessBase64ImageType(base64);
|
||||
|
||||
return {
|
||||
...item,
|
||||
image_url: {
|
||||
...item.image_url,
|
||||
url: `data:${imageType};base64,${base64}`
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return item;
|
||||
})
|
||||
);
|
||||
|
||||
return result.flat().filter(Boolean);
|
||||
return loadImageToBase64(result.flat().filter(Boolean) as ChatCompletionContentPart[]);
|
||||
};
|
||||
|
||||
// format GPT messages, concat text messages
|
||||
const clearInvalidMessages = (messages: ChatCompletionMessageParam[]) => {
|
||||
return messages
|
||||
@@ -247,7 +264,7 @@ export const loadRequestMessages = async ({
|
||||
};
|
||||
|
||||
if (messages.length === 0) {
|
||||
return Promise.reject('core.chat.error.Messages empty');
|
||||
return Promise.reject(i18nT('common:core.chat.error.Messages empty'));
|
||||
}
|
||||
|
||||
// filter messages file
|
||||
@@ -275,6 +292,7 @@ export const loadRequestMessages = async ({
|
||||
content: await parseUserContent(item.content)
|
||||
};
|
||||
} else if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
|
||||
// remove invalid field
|
||||
return {
|
||||
role: item.role,
|
||||
content: item.content,
|
||||
|
@@ -19,6 +19,7 @@
|
||||
"input_guide": "Input Guide",
|
||||
"input_guide_lexicon": "Lexicon",
|
||||
"input_guide_tip": "You can set up some preset questions. When the user inputs a question, related questions from these presets will be suggested.",
|
||||
"input_placeholder_phone": "Please enter your question",
|
||||
"insert_input_guide,_some_data_already_exists": "Duplicate data detected, automatically filtered, {{len}} items inserted",
|
||||
"is_chatting": "Chatting in progress... please wait until it finishes",
|
||||
"items": "Items",
|
||||
@@ -36,4 +37,4 @@
|
||||
"stream_output": "Stream Output",
|
||||
"view_citations": "View References",
|
||||
"web_site_sync": "Web Site Sync"
|
||||
}
|
||||
}
|
||||
|
@@ -19,6 +19,7 @@
|
||||
"input_guide": "输入引导",
|
||||
"input_guide_lexicon": "词库",
|
||||
"input_guide_tip": "可以配置一些预设的问题。在用户输入问题时,会从这些预设问题中获取相关问题进行提示。",
|
||||
"input_placeholder_phone": "输入问题",
|
||||
"insert_input_guide,_some_data_already_exists": "有重复数据,已自动过滤,共插入 {{len}} 条数据",
|
||||
"is_chatting": "正在聊天中...请等待结束",
|
||||
"items": "条",
|
||||
@@ -36,4 +37,4 @@
|
||||
"stream_output": "流输出",
|
||||
"view_citations": "查看引用",
|
||||
"web_site_sync": "Web站点同步"
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user