优化接受gpts模型，并支持上传文件

Dockerfile

@@ -19,6 +19,7 @@ ENV OPENAI_API_KEY=""
 ENV GOOGLE_API_KEY=""
 ENV CODE=""
 ENV NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN=1
+ENV NEXT_PUBLIC_ENABLE_BASE64=0
 
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules
@@ -35,6 +36,7 @@ ENV PROXY_URL=""
 ENV OPENAI_API_KEY=""
 ENV GOOGLE_API_KEY=""
 ENV CODE=""
+ENV NEXT_PUBLIC_ENABLE_BASE64=0
 
 COPY --from=builder /app/public ./public
 COPY --from=builder /app/.next/standalone ./

app/api/file/[...path]/route.ts

@@ -2,72 +2,7 @@ import { getServerSideConfig } from "@/app/config/server";
 import LocalFileStorage from "@/app/utils/local_file_storage";
 import S3FileStorage from "@/app/utils/s3_file_storage";
 import { NextRequest, NextResponse } from "next/server";
-
+import mime from 'mime';
-
-interface MimeTypeMap {
-  [extension: string]: string;
-}
-
-// 创建一个文件扩展名到MIME类型的映射
-const mimeTypeMap: MimeTypeMap = {
-  'png': 'image/png',
-  'jpg': 'image/jpeg',
-  'webp': 'image/webp',
-  'gif': 'image/gif',
-  'bmp': 'image/bmp',
-  'svg': 'image/svg+xml',
-  'txt': 'text/plain',
-  'pdf': 'application/pdf',
-  'doc': 'application/msword',
-  'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-  'xls': 'application/vnd.ms-excel',
-  'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-  'ppt': 'application/vnd.ms-powerpoint',
-  'pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-  'zip': 'application/zip',
-  'rar': 'application/x-rar-compressed',
-  'bin': 'application/octet-stream',
-
-  // Audio
-  'mp3': 'audio/mpeg',
-  'wav': 'audio/wav',
-  'ogg': 'audio/ogg',
-  'flac': 'audio/flac',
-  'aac': 'audio/aac',
-  'weba': 'audio/webm',
-  'midi': 'audio/midi',
-
-  // Video
-  'mp4': 'video/mp4',
-  'webm': 'video/webm',
-  'avi': 'video/x-msvideo',
-  'wmv': 'video/x-ms-wmv',
-  'flv': 'video/x-flv',
-  '3gp': 'video/3gpp',
-  'mkv': 'video/x-matroska',
-
-  //编程
-  'js': 'application/javascript',
-  'json': 'application/json',
-  'html': 'text/html',
-  'css': 'text/css',
-  'xml': 'application/xml',
-  'csv': 'text/csv',
-  'ts': 'text/typescript',
-  'java': 'text/x-java-source',
-  'py': 'text/x-python',
-  'c': 'text/x-csrc',
-  'cpp': 'text/x-c++src',
-  'h': 'text/x-chdr',
-  'hpp': 'text/x-c++hdr',
-  'php': 'application/x-httpd-php',
-  'rb': 'text/x-ruby',
-  'go': 'text/x-go',
-  'rs': 'text/rust',
-  'swift': 'text/x-swift',
-  'kt': 'text/x-kotlin',
-  'scala': 'text/x-scala',
-};
 
 function getMimeType(filePath: string): string {
   if (typeof filePath !== 'string' || filePath.trim() === '') {
@@ -75,11 +10,11 @@ function getMimeType(filePath: string): string {
   }
   const extension = filePath.split('.').pop();
   if (extension) {
-    return mimeTypeMap[extension] || 'application/octet-stream';
+    const mimeType = mime.getType(extension);
+    return mimeType || 'application/octet-stream';
   } else {
     return 'application/octet-stream';
   }
-
 }
 
 async function handle(
@@ -121,4 +56,3 @@ export const GET = handle;
 
 export const runtime = "nodejs";
 export const revalidate = 0;
-

app/client/platforms/google.ts

@@ -36,7 +36,7 @@ export class GeminiProApi implements LLMApi {
   }
   async chat(options: ChatOptions): Promise<void> {
     const messages: any[] = [];
-    if (options.config.model.includes("vision")) {
+    if (options.config.model.includes("vision") || options.config.model.includes("gizmo")) {
       for (const v of options.messages) {
         let message: any = {
           role: v.role.replace("assistant", "model").replace("system", "user"),

app/client/platforms/openai.ts

@@ -1,4 +1,3 @@
-"use client";
 import {
   ApiPath,
   DEFAULT_API_HOST,
@@ -26,6 +25,7 @@ import { prettyObject } from "@/app/utils/format";
 import { getClientConfig } from "@/app/config/client";
 import { makeAzurePath } from "@/app/azure";
 import axios from "axios";
+import mime from 'mime';
 
 export interface OpenAIListModelResponse {
   object: string;
@@ -40,6 +40,7 @@ export class ChatGPTApi implements LLMApi {
   private disableListModels = true;
 
   path(path: string, model?: string): string {
+
     const accessStore = useAccessStore.getState();
 
     const isAzure = accessStore.provider === ServiceProvider.Azure;
@@ -54,9 +55,7 @@ export class ChatGPTApi implements LLMApi {
 
     if (baseUrl.length === 0) {
       const isApp = !!getClientConfig()?.isApp;
-      baseUrl = isApp
+      baseUrl = isApp ? DEFAULT_API_HOST : ApiPath.OpenAI;
-        ? DEFAULT_API_HOST + "/proxy" + ApiPath.OpenAI
-        : ApiPath.OpenAI;
     }
 
     if (baseUrl.endsWith("/")) {
@@ -71,8 +70,6 @@ export class ChatGPTApi implements LLMApi {
       return [baseUrl, model, path].join("/");
     }
 
-    console.log("[Proxy Endpoint] ", baseUrl, path);
-
     return [baseUrl, path].join("/");
   }
 
@@ -88,7 +85,7 @@ export class ChatGPTApi implements LLMApi {
       const base64 = Buffer.from(response.data, "binary").toString("base64");
       return base64;
     };
-    if (options.config.model.includes("vision")) {
+    if (options.config.model.includes("vision") || options.config.model.includes("gizmo")) {
       for (const v of options.messages) {
         let message: {
           role: string;
@@ -107,79 +104,15 @@ export class ChatGPTApi implements LLMApi {
         });
         if (v.image_url) {
           let image_url_data = "";
-          if (process.env.NEXT_PUBLIC_ENABLE_BASE64 == '1') {
+          if (process.env.NEXT_PUBLIC_ENABLE_BASE64) {
             var base64Data = await getImageBase64Data(v.image_url);
-            interface MIMEMap {
+            let mimeType: string | null;
-              [key: string]: string;
-            }
-
-            const extensionToMIME: MIMEMap = {
-              'png': 'image/png',
-              'jpg': 'image/jpeg',
-              'webp': 'image/webp',
-              'gif': 'image/gif',
-              'bmp': 'image/bmp',
-              'svg': 'image/svg+xml',
-              'txt': 'text/plain',
-              'pdf': 'application/pdf',
-              'doc': 'application/msword',
-              'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-              'xls': 'application/vnd.ms-excel',
-              'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-              'ppt': 'application/vnd.ms-powerpoint',
-              'pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-              'zip': 'application/zip',
-              'rar': 'application/x-rar-compressed',
-              'bin': 'application/octet-stream',
-
-              // Audio
-              'mp3': 'audio/mpeg',
-              'wav': 'audio/wav',
-              'ogg': 'audio/ogg',
-              'flac': 'audio/flac',
-              'aac': 'audio/aac',
-              'weba': 'audio/webm',
-              'midi': 'audio/midi',
-
-              // Video
-              'mp4': 'video/mp4',
-              'webm': 'video/webm',
-              'avi': 'video/x-msvideo',
-              'wmv': 'video/x-ms-wmv',
-              'flv': 'video/x-flv',
-              '3gp': 'video/3gpp',
-              'mkv': 'video/x-matroska',
-
-              //编程
-              'js': 'application/javascript',
-              'json': 'application/json',
-              'html': 'text/html',
-              'css': 'text/css',
-              'xml': 'application/xml',
-              'csv': 'text/csv',
-              'ts': 'text/typescript',
-              'java': 'text/x-java-source',
-              'py': 'text/x-python',
-              'c': 'text/x-csrc',
-              'cpp': 'text/x-c++src',
-              'h': 'text/x-chdr',
-              'hpp': 'text/x-c++hdr',
-              'php': 'application/x-httpd-php',
-              'rb': 'text/x-ruby',
-              'go': 'text/x-go',
-              'rs': 'text/rust',
-              'swift': 'text/x-swift',
-              'kt': 'text/x-kotlin',
-              'scala': 'text/x-scala',
-            };
-
-            let mimeType: string | undefined;
             try {
               // 使用正则表达式获取文件后缀
               const match = v.image_url.match(/\.(\w+)$/);
-              if (match) {
+              if (match && match[1]) {
                 const fileExtension = match[1].toLowerCase();
-                mimeType = extensionToMIME[fileExtension];
+                mimeType = mime.getType(fileExtension);
                 if (!mimeType) {
                   throw new Error('Unknown file extension: ' + fileExtension);
                 }
@@ -193,6 +126,11 @@ export class ChatGPTApi implements LLMApi {
             image_url_data = `data:${mimeType};base64,${base64Data}`
           }
           else {
+            const match = v.image_url.match(/\.(\w+)$/);
+            if (match && match[1]) {
+                const fileExtension = match[1].toLowerCase();
+                v.image_url = v.image_url.replace(/\.\w+$/, '.' + fileExtension);
+            }
             var port = window.location.port ? ':' + window.location.port : '';
             var url = window.location.protocol + "//" + window.location.hostname + port;
             image_url_data = encodeURI(`${url}${v.image_url}`)
@@ -214,6 +152,7 @@ export class ChatGPTApi implements LLMApi {
         }),
       );
     }
+
     const modelConfig = {
       ...useAppConfig.getState().modelConfig,
       ...useChatStore.getState().currentSession().mask.modelConfig,
@@ -230,7 +169,7 @@ export class ChatGPTApi implements LLMApi {
       frequency_penalty: modelConfig.frequency_penalty,
       top_p: modelConfig.top_p,
       max_tokens:
-        modelConfig.model.includes("vision")
+        modelConfig.model.includes("vision") || modelConfig.model.includes("gizmo")
           ? modelConfig.max_tokens
           : null,
       // max_tokens: Math.max(modelConfig.max_tokens, 1024),

app/components/chat.tsx

@@ -538,7 +538,7 @@ export function ChatActions(props: {
         }
       }
     };
-    if (currentModel.includes("vision")) {
+    if (currentModel.includes("vision") || currentModel.includes("gizmo")) {
       window.addEventListener("paste", onPaste);
       return () => {
         window.removeEventListener("paste", onPaste);
@@ -609,7 +609,7 @@ export function ChatActions(props: {
 
         {config.pluginConfig.enable &&
           /^gpt(?!.*03\d{2}$).*$/.test(currentModel) &&
-          currentModel != "gpt-4-vision-preview" && (
+          (!currentModel.includes("vision") && !currentModel.includes("gizmo")) && (
             <ChatAction
               onClick={switchUsePlugins}
               text={
@@ -620,7 +620,7 @@ export function ChatActions(props: {
               icon={usePlugins ? <EnablePluginIcon /> : <DisablePluginIcon />}
             />
           )}
-        {currentModel.includes("vision") && (
+        {(currentModel.includes("vision") || currentModel.includes("gizmo")) && (
           <ChatAction
             onClick={selectImage}
             text="选择图片"
@@ -1412,7 +1412,7 @@ function _Chat() {
                       defaultShow={i >= messages.length - 6}
                     />
                   </div>
-                  {!isUser && message.model?.includes("vision") && (
+                  {!isUser && (message.model?.includes("vision") || message.model?.includes("gizmo")) && (
                     <div
                       className={[
                         styles["chat-message-actions"],

app/layout.tsx

@@ -7,6 +7,7 @@ import { type Metadata } from "next";
 import { SpeedInsights } from "@vercel/speed-insights/next";
 import { getServerSideConfig } from "./config/server";
 import { GoogleTagManager } from "@next/third-parties/google";
+import { Analytics } from "@vercel/analytics/react"
 const serverConfig = getServerSideConfig();
 
 export const metadata: Metadata = {

app/store/chat.ts

@@ -340,7 +340,7 @@ export const useChatStore = createPersistStore(
           session.mask.usePlugins &&
           allPlugins.length > 0 &&
           modelConfig.model.startsWith("gpt") &&
-          modelConfig.model != "gpt-4-vision-preview"
+          (!modelConfig.model.includes("vision") && !modelConfig.model.includes("gizmo"))
         ) {
           console.log("[ToolAgent] start");
           const pluginToolNames = allPlugins.map((m) => m.toolName);

package.json

@@ -20,7 +20,6 @@
     "@aws-sdk/s3-request-presigner": "^3.414.0",
     "@fortaine/fetch-event-source": "^3.0.6",
     "@hello-pangea/dnd": "^16.5.0",
-    "langchain": "0.1.20",
     "@langchain/community": "0.0.30",
     "@langchain/openai": "0.0.14",
     "@next/third-parties": "^14.1.0",
@@ -37,7 +36,9 @@
     "html-to-image": "^1.11.11",
     "html-to-text": "^9.0.5",
     "https-proxy-agent": "^7.0.2",
+    "langchain": "0.1.20",
     "mermaid": "^10.6.1",
+    "mime": "^4.0.1",
     "nanoid": "^5.0.3",
     "next": "^13.4.9",
     "node-fetch": "^3.3.1",