Merge pull request #5997 from ChatGPTNextWeb/feature/glm-4v

feature: support glm-4v

app/client/platforms/glm.ts

@@ -21,9 +21,10 @@ import {
   SpeechOptions,
 } from "../api";
 import { getClientConfig } from "@/app/config/client";
-import { getMessageTextContent } from "@/app/utils";
+import { getMessageTextContent, isVisionModel } from "@/app/utils";
 import { RequestPayload } from "./openai";
 import { fetch } from "@/app/utils/stream";
+import { preProcessImageContent } from "@/app/utils/chat";
 
 interface BasePayload {
   model: string;
@@ -154,9 +155,12 @@ export class ChatGLMApi implements LLMApi {
   }
 
   async chat(options: ChatOptions) {
+    const visionModel = isVisionModel(options.config.model);
     const messages: ChatOptions["messages"] = [];
     for (const v of options.messages) {
-      const content = getMessageTextContent(v);
+      const content = visionModel
+        ? await preProcessImageContent(v.content)
+        : getMessageTextContent(v);
       messages.push({ role: v.role, content });
     }
 
@@ -168,7 +172,6 @@ export class ChatGLMApi implements LLMApi {
         providerName: options.config.providerName,
       },
     };
-
     const modelType = this.getModelType(modelConfig.model);
     const requestPayload = this.createPayload(messages, modelConfig, options);
     const path = this.path(this.getModelPath(modelType));

app/constant.ts

@@ -305,6 +305,9 @@ export const VISION_MODEL_REGEXES = [
   /qwen2-vl/,
   /gpt-4-turbo(?!.*preview)/, // Matches "gpt-4-turbo" but not "gpt-4-turbo-preview"
   /^dall-e-3$/, // Matches exactly "dall-e-3"
+  /glm-4v-plus/,
+  /glm-4v/,
+  /glm-4v-flash/,
 ];
 
 export const EXCLUDE_VISION_MODEL_REGEXES = [/claude-3-5-haiku-20241022/];