Add vLLM as multimodal captioning source

003066a036f8640d875b44230b32b585e82445fc

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: a287ccfca268aae06d7c1e4a0cf38ddf8e320c9c
4 files changed, +26 -2Ignore whitespace
public/scripts/extensions/caption/index.js+1 -0
@@ -356,6 +356,7 @@ jQuery(async function () {
356 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||356 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
357 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||357 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
358 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'koboldcpp' && textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) ||358 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'koboldcpp' && textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) ||
359 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'vllm' && textgenerationwebui_settings.server_urls[textgen_types.VLLM]) ||
359 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'custom') ||360 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'custom') ||
360 extension_settings.caption.source === 'local' ||361 extension_settings.caption.source === 'local' ||
361 extension_settings.caption.source === 'horde';362 extension_settings.caption.source === 'horde';
public/scripts/extensions/caption/settings.html+2 -0
@@ -26,6 +26,7 @@
26 <option value="openai">OpenAI</option>26 <option value="openai">OpenAI</option>
27 <option value="openrouter">OpenRouter</option>27 <option value="openrouter">OpenRouter</option>
28 <option value="ooba" data-i18n="Text Generation WebUI (oobabooga)">Text Generation WebUI (oobabooga)</option>28 <option value="ooba" data-i18n="Text Generation WebUI (oobabooga)">Text Generation WebUI (oobabooga)</option>
29 <option value="vllm">vLLM</option>
29 </select>30 </select>
30 </div>31 </div>
31 <div class="flex1 flex-container flexFlowColumn flexNoGap">32 <div class="flex1 flex-container flexFlowColumn flexNoGap">
@@ -66,6 +67,7 @@
66 <option data-type="llamacpp" value="llamacpp_current" data-i18n="currently_loaded">[Currently loaded]</option>67 <option data-type="llamacpp" value="llamacpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
67 <option data-type="ooba" value="ooba_current" data-i18n="currently_loaded">[Currently loaded]</option>68 <option data-type="ooba" value="ooba_current" data-i18n="currently_loaded">[Currently loaded]</option>
68 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>69 <option data-type="koboldcpp" value="koboldcpp_current" data-i18n="currently_loaded">[Currently loaded]</option>
70 <option data-type="vllm" value="vllm_current" data-i18n="currently_selected">[Currently selected]</option>
69 <option data-type="custom" value="custom_current" data-i18n="currently_selected">[Currently selected]</option>71 <option data-type="custom" value="custom_current" data-i18n="currently_selected">[Currently selected]</option>
70 </select>72 </select>
71 </div>73 </div>
public/scripts/extensions/shared.js+17 -0
@@ -34,6 +34,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
34 const isCustom = extension_settings.caption.multimodal_api === 'custom';34 const isCustom = extension_settings.caption.multimodal_api === 'custom';
35 const isOoba = extension_settings.caption.multimodal_api === 'ooba';35 const isOoba = extension_settings.caption.multimodal_api === 'ooba';
36 const isKoboldCpp = extension_settings.caption.multimodal_api === 'koboldcpp';36 const isKoboldCpp = extension_settings.caption.multimodal_api === 'koboldcpp';
37 const isVllm = extension_settings.caption.multimodal_api === 'vllm';
37 const base64Bytes = base64Img.length * 0.75;38 const base64Bytes = base64Img.length * 0.75;
38 const compressionLimit = 2 * 1024 * 1024;39 const compressionLimit = 2 * 1024 * 1024;
39 if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {40 if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
@@ -65,6 +66,14 @@ export async function getMultimodalCaption(base64Img, prompt) {
65 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];66 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];
66 }67 }
6768
69 if (isVllm) {
70 if (extension_settings.caption.multimodal_model === 'vllm_current') {
71 requestBody.model = textgenerationwebui_settings.vllm_model;
72 }
73
74 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.VLLM];
75 }
76
68 if (isLlamaCpp) {77 if (isLlamaCpp) {
69 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP];78 requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP];
70 }79 }
@@ -151,6 +160,14 @@ function throwIfInvalidModel(useReverseProxy) {
151 throw new Error('KoboldCpp server URL is not set.');160 throw new Error('KoboldCpp server URL is not set.');
152 }161 }
153162
163 if (extension_settings.caption.multimodal_api === 'vllm' && !textgenerationwebui_settings.server_urls[textgen_types.VLLM]) {
164 throw new Error('vLLM server URL is not set.');
165 }
166
167 if (extension_settings.caption.multimodal_api === 'vllm' && extension_settings.caption.multimodal_model === 'vllm_current' && !textgenerationwebui_settings.vllm_model) {
168 throw new Error('vLLM model is not set.');
169 }
170
154 if (extension_settings.caption.multimodal_api === 'custom' && !oai_settings.custom_url) {171 if (extension_settings.caption.multimodal_api === 'custom' && !oai_settings.custom_url) {
155 throw new Error('Custom API URL is not set.');172 throw new Error('Custom API URL is not set.');
156 }173 }
src/endpoints/openai.js+6 -2
@@ -43,7 +43,11 @@ router.post('/caption-image', jsonParser, async (request, response) => {
43 key = readSecret(request.user.directories, SECRET_KEYS.KOBOLDCPP);43 key = readSecret(request.user.directories, SECRET_KEYS.KOBOLDCPP);
44 }44 }
4545
46 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp'].includes(request.body.api) === false) {46 if (request.body.api === 'vllm') {
47 key = readSecret(request.user.directories, SECRET_KEYS.VLLM);
48 }
49
50 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {
47 console.log('No key found for API', request.body.api);51 console.log('No key found for API', request.body.api);
48 return response.sendStatus(400);52 return response.sendStatus(400);
49 }53 }
@@ -110,7 +114,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
110 });114 });
111 }115 }
112116
113 if (request.body.api === 'koboldcpp') {117 if (request.body.api === 'koboldcpp' || request.body.api === 'vllm') {
114 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;118 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
115 }119 }
116120