Z.AI: Video inlining and 'coding' captions Closes #4899
| @@ -1998,7 +1998,7 @@ | ||
| 1998 | 1998 | <i class="icon-supported fa-solid fa-image" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i> |
| 1999 | 1999 | <i class="icon-unsupported fa-solid fa-image" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i> |
| 2000 | 2000 | </div> |
| 2001 | 2001 | <div id="openai_video_inlining_supported" data-source="makersuite,vertexai,openrouter,zai"> |
| 2002 | 2002 | <i class="icon-supported fa-solid fa-film" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i> |
| 2003 | 2003 | <i class="icon-unsupported fa-solid fa-film" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i> |
| 2004 | 2004 | </div> |
| @@ -455,7 +455,7 @@ function isVideoCaptioningAvailable() { | ||
| 455 | 455 | return false; |
| 456 | 456 | } |
| 457 | 457 | |
| 458 | 458 | return ['google', 'vertexai', 'zai'].includes(extension_settings.caption.multimodal_api); |
| 459 | 459 | } |
| 460 | 460 | |
| 461 | 461 | jQuery(async function () { |
| @@ -177,9 +177,6 @@ | ||
| 177 | 177 | <option data-type="custom" value="custom_current" data-i18n="currently_selected">[Currently selected]</option> |
| 178 | 178 | </select> |
| 179 | 179 | </div> |
| 180 | - <div data-type="zai"> | |
| 181 | - <b>Will use Common API. Coding API is not supported!</b> | |
| 182 | - </div> | |
| 183 | 180 | <div data-type="ollama"> |
| 184 | 181 | <div> |
| 185 | 182 | The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>. |
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | import { CONNECT_API_MAP, getRequestHeaders } from '../../script.js'; |
| 2 | 2 | import { extension_settings, openThirdPartyExtensionMenu } from '../extensions.js'; |
| 3 | 3 | import { t } from '../i18n.js'; |
| 4 | 4 | import { oai_settings, proxies, ZAI_ENDPOINT } from '../openai.js'; |
| 5 | 5 | import { SECRET_KEYS, secret_state } from '../secrets.js'; |
| 6 | 6 | import { textgen_types, textgenerationwebui_settings } from '../textgen-settings.js'; |
| 7 | 7 | import { getTokenCountAsync } from '../tokenizers.js'; |
| @@ -115,6 +115,10 @@ export async function getMultimodalCaption(base64Img, prompt) { | ||
| 115 | 115 | requestBody.custom_exclude_body = oai_settings.custom_exclude_body; |
| 116 | 116 | } |
| 117 | 117 | |
| 118 | + if (extension_settings.caption.multimodal_api === 'zai') { | |
| 119 | + requestBody.zai_endpoint = oai_settings.zai_endpoint || ZAI_ENDPOINT.COMMON; | |
| 120 | + } | |
| 121 | + | |
| 118 | 122 | function getEndpointUrl() { |
| 119 | 123 | switch (extension_settings.caption.multimodal_api) { |
| 120 | 124 | case 'google': |
| @@ -5870,12 +5870,15 @@ export function isVideoInliningSupported() { | ||
| 5870 | 5870 | return false; |
| 5871 | 5871 | } |
| 5872 | 5872 | |
| 5873 | - // Only Gemini models support video for now | |
| 5874 | 5873 | const videoSupportedModels = [ |
| 5874 | + // Gemini | |
| 5875 | 5875 | 'gemini-2.0', |
| 5876 | 5876 | 'gemini-2.5', |
| 5877 | 5877 | 'gemini-exp-1206', |
| 5878 | 5878 | 'gemini-3', |
| 5879 | + // Z.AI (GLM) | |
| 5880 | + 'glm-4.5v', | |
| 5881 | + 'glm-4.6v', | |
| 5879 | 5882 | ]; |
| 5880 | 5883 | |
| 5881 | 5884 | switch (oai_settings.chat_completion_source) { |
| @@ -5885,6 +5888,8 @@ export function isVideoInliningSupported() { | ||
| 5885 | 5888 | return videoSupportedModels.some(model => oai_settings.vertexai_model.includes(model)); |
| 5886 | 5889 | case chat_completion_sources.OPENROUTER: |
| 5887 | 5890 | return (Array.isArray(model_list) && model_list.find(m => m.id === oai_settings.openrouter_model)?.architecture?.input_modalities?.includes('video')); |
| 5891 | + case chat_completion_sources.ZAI: | |
| 5892 | + return videoSupportedModels.some(model => oai_settings.zai_model.includes(model)); | |
| 5888 | 5893 | default: |
| 5889 | 5894 | return false; |
| 5890 | 5895 | } |
| @@ -8,7 +8,7 @@ import express from 'express'; | ||
| 8 | 8 | import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1, delay } from '../util.js'; |
| 9 | 9 | import { setAdditionalHeaders } from '../additional-headers.js'; |
| 10 | 10 | import { readSecret, SECRET_KEYS } from './secrets.js'; |
| 11 | 11 | import { AIMLAPI_HEADERS, OPENROUTER_HEADERS, ZAI_ENDPOINT } from '../constants.js'; |
| 12 | 12 | |
| 13 | 13 | export const router = express.Router(); |
| 14 | 14 | |
| @@ -191,7 +191,22 @@ router.post('/caption-image', async (request, response) => { | ||
| 191 | 191 | } |
| 192 | 192 | |
| 193 | 193 | if (request.body.api === 'zai') { |
| 194 | - apiUrl = 'https://api.z.ai/api/paas/v4/chat/completions'; | |
| 194 | + apiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING | |
| 195 | + ? 'https://api.z.ai/api/coding/paas/v4/chat/completions' | |
| 196 | + : 'https://api.z.ai/api/paas/v4/chat/completions'; | |
| 197 | + | |
| 198 | + // Handle video inlining for Z.AI | |
| 199 | + if (/data:video\/\w+;base64,/.test(request.body.image)) { | |
| 200 | + const message = body.messages.find(msg => Array.isArray(msg.content)); | |
| 201 | + if (message) { | |
| 202 | + const imgContent = message.content.find(c => c.type === 'image_url'); | |
| 203 | + if (imgContent) { | |
| 204 | + imgContent.type = 'video_url'; | |
| 205 | + imgContent.video_url = imgContent.image_url; | |
| 206 | + delete imgContent.image_url; | |
| 207 | + } | |
| 208 | + } | |
| 209 | + } | |
| 195 | 210 | } |
| 196 | 211 | |
| 197 | 212 | if (['koboldcpp', 'vllm', 'llamacpp', 'ooba'].includes(request.body.api)) { |