c92939e5 · permissionBRICK/SillyTavern

public/index.html+1 -1 Viewed

                                                     <i class="icon-supported fa-solid fa-image" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
                                                     <i class="icon-unsupported fa-solid fa-image" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
                                                 </div>
-                                                <div id="openai_video_inlining_supported" data-source="makersuite,vertexai,openrouter">
+                                                <div id="openai_video_inlining_supported" data-source="makersuite,vertexai,openrouter,zai">
                                                     <i class="icon-supported fa-solid fa-film" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
                                                     <i class="icon-unsupported fa-solid fa-film" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
                                                 </div>

public/scripts/extensions/caption/index.js+1 -1 Viewed

@@ -455,7 +455,7 @@ function isVideoCaptioningAvailable() {
455	return false;	455	return false;
456	}	456	}
457		457
458	return ['google', 'vertexai'].includes(extension_settings.caption.multimodal_api);	458	return ['google', 'vertexai', 'zai'].includes(extension_settings.caption.multimodal_api);
459	}	459	}
460		460
461	jQuery(async function () {	461	jQuery(async function () {

public/scripts/extensions/caption/settings.html+0 -3 Viewed

                         <option data-type="custom" value="custom_current" data-i18n="currently_selected">[Currently selected]</option>
                     </select>
                 </div>
-                <div data-type="zai">
-                    <b>Will use Common API. Coding API is not supported!</b>
-                </div>
                 <div data-type="ollama">
                     <div>
                         The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.

public/scripts/extensions/shared.js+5 -1 Viewed

 import { CONNECT_API_MAP, getRequestHeaders } from '../../script.js';
 import { extension_settings, openThirdPartyExtensionMenu } from '../extensions.js';
 import { t } from '../i18n.js';
-import { oai_settings, proxies } from '../openai.js';
+import { oai_settings, proxies, ZAI_ENDPOINT } from '../openai.js';
 import { SECRET_KEYS, secret_state } from '../secrets.js';
 import { textgen_types, textgenerationwebui_settings } from '../textgen-settings.js';
 import { getTokenCountAsync } from '../tokenizers.js';

         requestBody.custom_exclude_body = oai_settings.custom_exclude_body;
+    }
+    if (extension_settings.caption.multimodal_api === 'zai') {
+        requestBody.zai_endpoint = oai_settings.zai_endpoint || ZAI_ENDPOINT.COMMON;
+    }
     function getEndpointUrl() {
         switch (extension_settings.caption.multimodal_api) {
             case 'google':

public/scripts/openai.js+6 -1 Viewed

         return false;
+    }
-    // Only Gemini models support video for now
     const videoSupportedModels = [
+        // Gemini
         'gemini-2.0',
         'gemini-2.5',
         'gemini-exp-1206',
         'gemini-3',
+        // Z.AI (GLM)
+        'glm-4.5v',
+        'glm-4.6v',
     ];
     switch (oai_settings.chat_completion_source) {

             return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
         case chat_completion_sources.OPENROUTER:
             return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('video'));
+        case chat_completion_sources.ZAI:
+            return videoSupportedModels.some(model => oai_settings.zai_model.includes(model));
         default:
             return false;
+    }

src/endpoints/openai.js+17 -2 Viewed

 import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1, delay } from '../util.js';
 import { setAdditionalHeaders } from '../additional-headers.js';
 import { readSecret, SECRET_KEYS } from './secrets.js';
-import { AIMLAPI_HEADERS, OPENROUTER_HEADERS } from '../constants.js';
+import { AIMLAPI_HEADERS, OPENROUTER_HEADERS, ZAI_ENDPOINT } from '../constants.js';
 export const router = express.Router();

+        }
         if (request.body.api === 'zai') {
-            apiUrl = 'https://api.z.ai/api/paas/v4/chat/completions';
+            apiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING
+                ? 'https://api.z.ai/api/coding/paas/v4/chat/completions'
+                : 'https://api.z.ai/api/paas/v4/chat/completions';
+            // Handle video inlining for Z.AI
+            if (/data:video\/\w+;base64,/.test(request.body.image)) {
+                const message = body.messages.find(msg => Array.isArray(msg.content));
+                if (message) {
+                    const imgContent = message.content.find(c => c.type === 'image_url');
+                    if (imgContent) {
+                        imgContent.type = 'video_url';
+                        imgContent.video_url = imgContent.image_url;
+                        delete imgContent.image_url;
+                    }
+                }
+            }
+        }
         if (['koboldcpp', 'vllm', 'llamacpp', 'ooba'].includes(request.body.api)) {

Z.AI: Video inlining and 'coding' captions Closes #4899