image compatibility for various content-types (#6119)

* image compatibility for various content-types

* perf: image type detect

* perf: gethistory

* update test

* update rerank log

* perf: login

* fix: query extension use

---------

Co-authored-by: archer <545436317@qq.com>

packages/service/core/ai/embedding/index.ts

@@ -3,6 +3,8 @@ import { getAIApi } from '../config';
 import { countPromptTokens } from '../../../common/string/tiktoken/index';
 import { EmbeddingTypeEnm } from '@fastgpt/global/core/ai/constants';
 import { addLog } from '../../../common/system/log';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { retryFn } from '@fastgpt/global/common/system/utils';
 
 type GetVectorProps = {
   model: EmbeddingModelItemType;
@@ -38,55 +40,70 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
 
     for (const chunk of chunks) {
       // input text to vector
-      const result = await ai.embeddings
-        .create(
-          {
-            ...model.defaultConfig,
-            ...(type === EmbeddingTypeEnm.db && model.dbConfig),
-            ...(type === EmbeddingTypeEnm.query && model.queryConfig),
-            model: model.model,
-            input: chunk
-          },
-          model.requestUrl
-            ? {
-                path: model.requestUrl,
-                headers: {
-                  ...(model.requestAuth ? { Authorization: `Bearer ${model.requestAuth}` } : {}),
-                  ...headers
+      const result = await retryFn(() =>
+        ai.embeddings
+          .create(
+            {
+              ...model.defaultConfig,
+              ...(type === EmbeddingTypeEnm.db && model.dbConfig),
+              ...(type === EmbeddingTypeEnm.query && model.queryConfig),
+              model: model.model,
+              input: chunk
+            },
+            model.requestUrl
+              ? {
+                  path: model.requestUrl,
+                  headers: {
+                    ...(model.requestAuth ? { Authorization: `Bearer ${model.requestAuth}` } : {}),
+                    ...headers
+                  }
                 }
-              }
-            : { headers }
-        )
-        .then(async (res) => {
-          if (!res.data) {
-            addLog.error('[Embedding] API is not responding', res);
-            return Promise.reject('Embedding API is not responding');
-          }
-          if (!res?.data?.[0]?.embedding) {
-            // @ts-ignore
-            const msg = res.data?.err?.message || 'Embedding API Error';
-            addLog.error('[Embedding] API Error', {
-              message: msg,
-              data: res
-            });
-            return Promise.reject(msg);
-          }
+              : { headers }
+          )
+          .then(async (res) => {
+            if (!res.data) {
+              addLog.error('[Embedding Error] not responding', {
+                message: '',
+                data: {
+                  response: res,
+                  model: model.model,
+                  inputLength: chunk.length
+                }
+              });
+              return Promise.reject('Embedding API is not responding');
+            }
+            if (!res?.data?.[0]?.embedding) {
+              // @ts-ignore
+              const msg = res.data?.err?.message || '';
+              addLog.error('[Embedding Error]', {
+                message: msg,
+                data: {
+                  response: res,
+                  model: model.model,
+                  inputLength: chunk.length
+                }
+              });
+              return Promise.reject('Embedding API is not responding');
+            }
 
-          const [tokens, vectors] = await Promise.all([
-            (async () => {
-              if (res.usage) return res.usage.total_tokens;
+            const [tokens, vectors] = await Promise.all([
+              (async () => {
+                if (res.usage) return res.usage.total_tokens;
 
-              const tokens = await Promise.all(chunk.map((item) => countPromptTokens(item)));
-              return tokens.reduce((sum, item) => sum + item, 0);
-            })(),
-            Promise.all(res.data.map((item) => formatVectors(item.embedding, model.normalization)))
-          ]);
+                const tokens = await Promise.all(chunk.map((item) => countPromptTokens(item)));
+                return tokens.reduce((sum, item) => sum + item, 0);
+              })(),
+              Promise.all(
+                res.data.map((item) => formatVectors(item.embedding, model.normalization))
+              )
+            ]);
 
-          return {
-            tokens,
-            vectors
-          };
-        });
+            return {
+              tokens,
+              vectors
+            };
+          })
+      );
 
       totalTokens += result.tokens;
       allVectors.push(...result.vectors);
@@ -97,7 +114,13 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
       vectors: allVectors
     };
   } catch (error) {
-    addLog.error(`[Embedding] request error`, error);
+    addLog.error(`[Embedding Error]`, {
+      message: getErrText(error),
+      data: {
+        model: model.model,
+        inputLengths: formatInput.map((item) => item.length)
+      }
+    });
 
     return Promise.reject(error);
   }

packages/service/core/ai/llm/request.ts

@@ -628,12 +628,17 @@ const createChatCompletion = async ({
       ('iterator' in response || 'controller' in response);
 
     const getEmptyResponseTip = () => {
-      addLog.warn(`LLM response empty`, {
-        baseUrl: userKey?.baseUrl,
-        requestBody: body
-      });
       if (userKey?.baseUrl) {
+        addLog.warn(`User LLM response empty`, {
+          baseUrl: userKey?.baseUrl,
+          requestBody: body
+        });
         return `您的 OpenAI key 没有响应: ${JSON.stringify(body)}`;
+      } else {
+        addLog.error(`LLM response empty`, {
+          message: '',
+          data: body
+        });
       }
       return i18nT('chat:LLM_model_response_empty');
     };
@@ -652,13 +657,18 @@ const createChatCompletion = async ({
       getEmptyResponseTip
     };
   } catch (error) {
-    addLog.error(`LLM response error`, error);
-    addLog.warn(`LLM response error`, {
-      baseUrl: userKey?.baseUrl,
-      requestBody: body
-    });
     if (userKey?.baseUrl) {
+      addLog.warn(`User ai api error`, {
+        message: getErrText(error),
+        baseUrl: userKey?.baseUrl,
+        data: body
+      });
       return Promise.reject(`您的 OpenAI key 出错了: ${getErrText(error)}`);
+    } else {
+      addLog.error(`LLM response error`, {
+        message: getErrText(error),
+        data: body
+      });
     }
     return Promise.reject(error);
   }

packages/service/core/ai/rerank/index.ts

@@ -35,7 +35,7 @@ export function reRankRecall({
   headers?: Record<string, string>;
 }): Promise<ReRankCallResult> {
   if (!model) {
-    return Promise.reject('[Rerank] No rerank model');
+    return Promise.reject('No rerank model');
   }
   if (documents.length === 0) {
     return Promise.resolve({
@@ -67,7 +67,7 @@ export function reRankRecall({
       addLog.info('ReRank finish:', { time: Date.now() - start });
 
       if (!data?.results || data?.results?.length === 0) {
-        addLog.error('[Rerank] Empty result', { data });
+        addLog.error('[Rerank Error]', { message: 'Empty result', data });
       }
 
       return {
@@ -81,7 +81,7 @@ export function reRankRecall({
       };
     })
     .catch((err) => {
-      addLog.error('[Rerank] request error', err);
+      addLog.error('[Rerank Error]', err);
 
       return Promise.reject(err);
     });