Add Vertex AI express mode support (#3977) * Add Vertex AI express mode support Split Google AI Studio and Vertex AI * Add support for Vertex AI, including updating default models and related settings, modifying frontend HTML to include Vertex AI options, and adjusting request processing logic in the backend API. * Log API name in the console * Merge sysprompt toggles back * Use Gemma tokenizers for Vertex and LearnLM * AI Studio parity updates * Add link to express mode doc. Also technically it's not a form * Split title * Use array includes * Add support for Google Vertex AI in image captioning feature * Specify caption API name, add to compression list --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

157315cd6890c3738b95f3d164ffc477e3469425

NijikaMyWaifu <139055015+NijikaMyWaifu@users.noreply.github.com>

Signed
Parents: 6dc59b9fd3a36e4abb2edc42e6e8a70e9d0e2100
18 files changed, +191 -40Showing whitespace changes
default/content/presets/openai/Default.json+1 -0
@@ -15,6 +15,7 @@
1515 "custom_exclude_body": "",
1616 "custom_include_headers": "",
1717 "google_model": "gemini-pro",
18+ "vertexai_model": "gemini-2.0-flash-001",
1819 "temperature": 1,
1920 "frequency_penalty": 0,
2021 "presence_penalty": 0,
public/index.html+46 -13
@@ -691,7 +691,7 @@
691691 </span>
692692 </div>
693693 </div>
694694 <div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale,makersuite,vertexai,mistralai,custom,cohere,perplexity,groq,01ai,nanogpt,deepseek,xai">
695695 <div class="range-block-title" data-i18n="Temperature">
696696 Temperature
697697 </div>
@@ -730,7 +730,7 @@
730730 </div>
731731 </div>
732732 </div>
733733 <div class="range-block" data-source="claude,openrouter,makersuite,vertexai,cohere,perplexity">
734734 <div class="range-block-title" data-i18n="Top K">
735735 Top K
736736 </div>
@@ -743,7 +743,7 @@
743743 </div>
744744 </div>
745745 </div>
746746 <div class="range-block" data-source="openai,claude,openrouter,ai21,scale,makersuite,vertexai,mistralai,custom,cohere,perplexity,groq,01ai,nanogpt,deepseek,xai">
747747 <div class="range-block-title" data-i18n="Top P">
748748 Top P
749749 </div>
@@ -1969,7 +1969,7 @@
19691969 </span>
19701970 </div>
19711971 </div>
19721972 <div class="range-block" data-source="makersuite,vertexai,openrouter,claude">
19731973 <label for="openai_enable_web_search" class="checkbox_label flexWrap widthFreeExpand">
19741974 <input id="openai_enable_web_search" type="checkbox" />
19751975 <span data-i18n="Enable web search">Enable web search</span>
@@ -1983,7 +1983,7 @@
19831983 </b>
19841984 </div>
19851985 </div>
19861986 <div class="range-block" data-source="openai,cohere,mistralai,custom,claude,openrouter,groq,deepseek,makersuite,vertexai,ai21,xai,pollinations">
19871987 <label for="openai_function_calling" class="checkbox_label flexWrap widthFreeExpand">
19881988 <input id="openai_function_calling" type="checkbox" />
19891989 <span data-i18n="Enable function calling">Enable function calling</span>
@@ -1993,7 +1993,7 @@
19931993 <span data-i18n="enable_functions_desc_3">Can be utilized by various extensions to provide additional functionality.</span>
19941994 </div>
19951995 </div>
19961996 <div class="range-block" data-source="openai,openrouter,mistralai,makersuite,vertexai,claude,custom,01ai,xai,pollinations">
19971997 <label for="openai_image_inlining" class="checkbox_label flexWrap widthFreeExpand">
19981998 <input id="openai_image_inlining" type="checkbox" />
19991999 <span data-i18n="Send inline images">Send inline images</span>
@@ -2018,7 +2018,7 @@
20182018 </div>
20192019 </div>
20202020 </div>
20212021 <div class="range-block" data-source="makersuite,vertexai">
20222022 <label for="openai_request_images" class="checkbox_label widthFreeExpand">
20232023 <input id="openai_request_images" type="checkbox" />
20242024 <span>
@@ -2030,12 +2030,12 @@
20302030 <span data-i18n="Allows the model to return image attachments.">
20312031 Allows the model to return image attachments.
20322032 </span>
20332033 <em data-source="makersuite,vertexai" data-i18n="Request inline images_desc_2">
20342034 Incompatible with the following features: function calling, web search, system prompt.
20352035 </em>
20362036 </div>
20372037 </div>
20382038 <div class="range-block" data-source="makersuite,vertexai">
20392039 <label for="use_makersuite_sysprompt" class="checkbox_label widthFreeExpand">
20402040 <input id="use_makersuite_sysprompt" type="checkbox" />
20412041 <span>
@@ -2060,12 +2060,12 @@
20602060 </span>
20612061 </div>
20622062 </div>
20632063 <div class="flex-container flexFlowColumn wide100p textAlignCenter marginTop10" data-source="openai,custom,claude,xai,makersuite,vertexai,openrouter,pollinations">
20642064 <div class="flex-container oneline-dropdown" title="Constrains effort on reasoning for reasoning models.&#10;Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response." data-i18n="[title]Constrains effort on reasoning for reasoning models.">
20652065 <label for="openai_reasoning_effort">
20662066 <span data-i18n="Reasoning Effort">Reasoning Effort</span>
20672067 <i data-source="openai,custom,xai,openrouter" class="opacity50p fa-solid fa-circle-info" title="OpenAI-style options: low, medium, high. Minimum and maximum are aliased to low and high. Auto does not send an effort level." data-i18n="[title]OpenAI-style options: low, medium, high. Minimum and maximum are aliased to low and high. Auto does not send an effort level."></i>
20682068 <i data-source="claude,makersuite,vertexai" class="opacity50p fa-solid fa-circle-info" title="Allocates a portion of the response length for thinking (low: 10%, medium: 25%, high: 50%). Other options are model-dependent." data-i18n="[title]Allocates a portion of the response length for thinking (low: 10%, medium: 25%, high: 50%). Other options are model-dependent."></i>
20692069 </label>
20702070 <select id="openai_reasoning_effort">
20712071 <option data-i18n="openai_reasoning_effort_auto" value="auto">Auto</option>
@@ -2769,6 +2769,7 @@
27692769 <option value="deepseek">DeepSeek</option>
27702770 <option value="groq">Groq</option>
27712771 <option value="makersuite">Google AI Studio</option>
2772+ <option value="vertexai">Google Vertex AI (Express mode)</option>
27722773 <option value="mistralai">MistralAI</option>
27732774 <option value="nanogpt">NanoGPT</option>
27742775 <option value="openrouter">OpenRouter</option>
@@ -2779,7 +2780,7 @@
27792780 <option value="xai">xAI (Grok)</option>
27802781 </optgroup>
27812782 </select>
27822783 <div class="inline-drawer wide100p" data-source="openai,claude,mistralai,makersuite,vertexai,deepseek,xai">
27832784 <div class="inline-drawer-toggle inline-drawer-header">
27842785 <b data-i18n="Reverse Proxy">Reverse Proxy</b>
27852786 <div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>
@@ -2843,7 +2844,7 @@
28432844 </div>
28442845 </div>
28452846 </div>
28462847 <div id="ReverseProxyWarningMessage" data-source="openai,claude,mistralai,makersuite,vertexai,deepseek,xai">
28472848 <div class="reverse_proxy_warning">
28482849 <b>
28492850 <div data-i18n="Using a proxy that you're not running yourself is a risk to your data privacy.">
@@ -3209,6 +3210,38 @@
32093210 </select>
32103211 </div>
32113212 </form>
3213+ <div id="vertexai_form" data-source="vertexai">
3214+ <h4>
3215+ <span data-i18n="Google Vertex AI API Key">
3216+ Google Vertex AI API Key
3217+ </span>
3218+ <a href="https://cloud.google.com/vertex-ai/generative-ai/docs/start/express-mode/overview" data-i18n="(Express mode keys only)" target="_blank" rel="noopener noreferrer">
3219+ (Express mode keys only)
3220+ </a>
3221+ </h4>
3222+ <div class="flex-container">
3223+ <input id="api_key_vertexai" name="api_key_vertexai" class="text_pole flex1" value="" type="text" autocomplete="off">
3224+ <div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_vertexai"></div>
3225+ </div>
3226+ <div data-for="api_key_vertexai" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
3227+ For privacy reasons, your API key will be hidden after you reload the page.
3228+ </div>
3229+ <div>
3230+ <h4 data-i18n="Google Model">Google Model</h4>
3231+ <select id="model_vertexai_select">
3232+ <optgroup label="Gemini 2.5">
3233+ <option value="gemini-2.5-pro-preview-05-06">gemini-2.5-pro-preview-05-06</option>
3234+ <option value="gemini-2.5-pro-preview-03-25">gemini-2.5-pro-preview-03-25</option>
3235+ <option value="gemini-2.5-flash-preview-05-20">gemini-2.5-flash-preview-05-20</option>
3236+ <option value="gemini-2.5-flash-preview-04-17">gemini-2.5-flash-preview-04-17</option>
3237+ </optgroup>
3238+ <optgroup label="Gemini 2.0">
3239+ <option value="gemini-2.0-flash-001">gemini-2.0-flash-001</option>
3240+ <option value="gemini-2.0-flash-lite-001">gemini-2.0-flash-lite-001</option>
3241+ </optgroup>
3242+ </select>
3243+ </div>
3244+ </div>
32123245 <form id="mistralai_form" data-source="mistralai" action="javascript:void(null);" method="post" enctype="multipart/form-data">
32133246 <h4 data-i18n="MistralAI API Key">MistralAI API Key</h4>
32143247 <div class="flex-container">
public/script.js+1 -0
@@ -5941,6 +5941,7 @@ function extractImageFromData(data, { mainApi = null, chatCompletionSource = nul
59415941 switch (mainApi ?? main_api) {
59425942 case 'openai': {
59435943 switch (chatCompletionSource ?? oai_settings.chat_completion_source) {
5944+ case chat_completion_sources.VERTEXAI:
59445945 case chat_completion_sources.MAKERSUITE: {
59455946 const inlineData = data?.responseContent?.parts?.find(x => x.inlineData)?.inlineData;
59465947 if (inlineData) {
public/scripts/RossAscends-mods.js+1 -0
@@ -402,6 +402,7 @@ function RA_autoconnect(PrevApi) {
402402 || (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER)
403403 || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21)
404404 || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE)
405+ || (secret_state[SECRET_KEYS.VERTEXAI] && oai_settings.chat_completion_source == chat_completion_sources.VERTEXAI)
405406 || (secret_state[SECRET_KEYS.MISTRALAI] && oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI)
406407 || (secret_state[SECRET_KEYS.COHERE] && oai_settings.chat_completion_source == chat_completion_sources.COHERE)
407408 || (secret_state[SECRET_KEYS.PERPLEXITY] && oai_settings.chat_completion_source == chat_completion_sources.PERPLEXITY)
public/scripts/extensions/caption/index.js+1 -0
@@ -414,6 +414,7 @@ jQuery(async function () {
414414 'openai': SECRET_KEYS.OPENAI,
415415 'mistral': SECRET_KEYS.MISTRALAI,
416416 'google': SECRET_KEYS.MAKERSUITE,
417+ 'vertexai': SECRET_KEYS.VERTEXAI,
417418 'anthropic': SECRET_KEYS.CLAUDE,
418419 };
419420
public/scripts/extensions/caption/settings.html+8 -1
@@ -22,6 +22,7 @@
2222 <option value="cohere">Cohere</option>
2323 <option value="custom" data-i18n="Custom (OpenAI-compatible)">Custom (OpenAI-compatible)</option>
2424 <option value="google">Google AI Studio</option>
25+ <option value="vertexai">Google Vertex AI</option>
2526 <option value="groq">Groq</option>
2627 <option value="koboldcpp">KoboldCpp</option>
2728 <option value="llamacpp">llama.cpp</option>
@@ -111,6 +112,12 @@
111112 <option data-type="google" value="gemini-1.5-flash-8b-exp-0827">gemini-1.5-flash-8b-exp-0827</option>
112113 <option data-type="google" value="learnlm-2.0-flash-experimental">learnlm-2.0-flash-experimental</option>
113114 <option data-type="google" value="learnlm-1.5-pro-experimental">learnlm-1.5-pro-experimental</option>
115+ <option data-type="vertexai" value="gemini-2.5-pro-preview-05-06">gemini-2.5-pro-preview-05-06</option>
116+ <option data-type="vertexai" value="gemini-2.5-pro-preview-03-25">gemini-2.5-pro-preview-03-25</option>
117+ <option data-type="vertexai" value="gemini-2.5-flash-preview-05-20">gemini-2.5-flash-preview-05-20</option>
118+ <option data-type="vertexai" value="gemini-2.5-flash-preview-04-17">gemini-2.5-flash-preview-04-17</option>
119+ <option data-type="vertexai" value="gemini-2.0-flash-001">gemini-2.0-flash-001</option>
120+ <option data-type="vertexai" value="gemini-2.0-flash-lite-001">gemini-2.0-flash-lite-001</option>
114121 <option data-type="groq" value="llama-3.2-11b-vision-preview">llama-3.2-11b-vision-preview</option>
115122 <option data-type="groq" value="llama-3.2-90b-vision-preview">llama-3.2-90b-vision-preview</option>
116123 <option data-type="groq" value="llava-v1.5-7b-4096-preview">llava-v1.5-7b-4096-preview</option>
@@ -170,7 +177,7 @@
170177 <div data-type="ollama">
171178 The model must be downloaded first! Do it with the <code>ollama pull</code> command or <a href="#" id="caption_ollama_pull">click here</a>.
172179 </div>
173180 <label data-type="openai,anthropic,google,vertexai,mistral" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
174181 <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
175182 <span data-i18n="Allow reverse proxy">Allow reverse proxy</span>
176183 </label>
public/scripts/extensions/shared.js+8 -2
@@ -15,7 +15,7 @@ import { createThumbnail, isValidUrl } from '../utils.js';
1515 */
1616export async function getMultimodalCaption(base64Img, prompt) {
1717 const useReverseProxy =
1818 (['openai', 'anthropic', 'google', 'mistral', 'vertexai'].includes(extension_settings.caption.multimodal_api))
1919 && extension_settings.caption.allow_reverse_proxy
2020 && oai_settings.reverse_proxy
2121 && isValidUrl(oai_settings.reverse_proxy);
@@ -38,7 +38,8 @@ export async function getMultimodalCaption(base64Img, prompt) {
3838 const isVllm = extension_settings.caption.multimodal_api === 'vllm';
3939 const base64Bytes = base64Img.length * 0.75;
4040 const compressionLimit = 2 * 1024 * 1024;
4141 ifconst ((thumbnailNeeded = ['google', 'openrouter', 'mistral', 'groq', 'vertexai'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {;
42+ if ((thumbnailNeeded && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
4243 const maxSide = 1024;
4344 base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg');
4445 }
@@ -94,6 +95,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
9495 function getEndpointUrl() {
9596 switch (extension_settings.caption.multimodal_api) {
9697 case 'google':
98+ case 'vertexai':
9799 return '/api/google/caption-image';
98100 case 'anthropic':
99101 return '/api/anthropic/caption-image';
@@ -143,6 +145,10 @@ function throwIfInvalidModel(useReverseProxy) {
143145 throw new Error('Google AI Studio API key is not set.');
144146 }
145147
148+ if (extension_settings.caption.multimodal_api === 'vertexai' && !secret_state[SECRET_KEYS.VERTEXAI] && !useReverseProxy) {
149+ throw new Error('Google Vertex AI API key is not set.');
150+ }
151+
146152 if (extension_settings.caption.multimodal_api === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI] && !useReverseProxy) {
147153 throw new Error('Mistral AI API key is not set.');
148154 }
public/scripts/openai.js+58 -7
@@ -176,6 +176,7 @@ export const chat_completion_sources = {
176176 OPENROUTER: 'openrouter',
177177 AI21: 'ai21',
178178 MAKERSUITE: 'makersuite',
179+ VERTEXAI: 'vertexai',
179180 MISTRALAI: 'mistralai',
180181 CUSTOM: 'custom',
181182 COHERE: 'cohere',
@@ -277,6 +278,7 @@ export const settingsToUpdate = {
277278 custom_include_headers: ['#custom_include_headers', 'custom_include_headers', false, true],
278279 custom_prompt_post_processing: ['#custom_prompt_post_processing', 'custom_prompt_post_processing', false, true],
279280 google_model: ['#model_google_select', 'google_model', false, true],
281+ vertexai_model: ['#model_vertexai_select', 'vertexai_model', false, true],
280282 openai_max_context: ['#openai_max_context', 'openai_max_context', false, false],
281283 openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false, false],
282284 wrap_in_quotes: ['#wrap_in_quotes', 'wrap_in_quotes', true, false],
@@ -350,6 +352,7 @@ const default_settings = {
350352 openai_model: 'gpt-4-turbo',
351353 claude_model: 'claude-3-5-sonnet-20240620',
352354 google_model: 'gemini-1.5-pro',
355+ vertexai_model: 'gemini-2.0-flash-001',
353356 ai21_model: 'jamba-1.6-large',
354357 mistralai_model: 'mistral-large-latest',
355358 cohere_model: 'command-r-plus',
@@ -433,6 +436,7 @@ const oai_settings = {
433436 openai_model: 'gpt-4-turbo',
434437 claude_model: 'claude-3-5-sonnet-20240620',
435438 google_model: 'gemini-1.5-pro',
439+ vertexai_model: 'gemini-2.0-flash-001',
436440 ai21_model: 'jamba-1.6-large',
437441 mistralai_model: 'mistral-large-latest',
438442 cohere_model: 'command-r-plus',
@@ -1666,6 +1670,8 @@ export function getChatCompletionModel(source = null) {
16661670 return '';
16671671 case chat_completion_sources.MAKERSUITE:
16681672 return oai_settings.google_model;
1673+ case chat_completion_sources.VERTEXAI:
1674+ return oai_settings.vertexai_model;
16691675 case chat_completion_sources.OPENROUTER:
16701676 return oai_settings.openrouter_model !== openrouter_website_model ? oai_settings.openrouter_model : null;
16711677 case chat_completion_sources.AI21:
@@ -2048,6 +2054,7 @@ async function sendOpenAIRequest(type, messages, signal) {
20482054 const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
20492055 const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
20502056 const isGoogle = oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE;
2057+ const isVertexAI = oai_settings.chat_completion_source == chat_completion_sources.VERTEXAI;
20512058 const isOAI = oai_settings.chat_completion_source == chat_completion_sources.OPENAI;
20522059 const isMistral = oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI;
20532060 const isCustom = oai_settings.chat_completion_source == chat_completion_sources.CUSTOM;
@@ -2123,8 +2130,8 @@ async function sendOpenAIRequest(type, messages, signal) {
21232130 delete generate_data.stop;
21242131 }
21252132
21262133 // Proxy is only supported for Claude, OpenAI, Mistral, and Google MakerSuite, and Vertex AI
21272134 if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI, chat_completion_sources.MISTRALAI, chat_completion_sources.MAKERSUITE, chat_completion_sources.VERTEXAI, chat_completion_sources.DEEPSEEK, chat_completion_sources.XAI].includes(oai_settings.chat_completion_source)) {
21282135 await validateReverseProxy();
21292136 generate_data['reverse_proxy'] = oai_settings.reverse_proxy;
21302137 generate_data['proxy_password'] = oai_settings.proxy_password;
@@ -2175,7 +2182,7 @@ async function sendOpenAIRequest(type, messages, signal) {
21752182 generate_data['api_url_scale'] = oai_settings.api_url_scale;
21762183 }
21772184
21782185 if (isGoogle || isVertexAI) {
21792186 const stopStringsLimit = 5;
21802187 generate_data['top_k'] = Number(oai_settings.top_k_openai);
21812188 generate_data['stop'] = getCustomStoppingStrings(stopStringsLimit).slice(0, stopStringsLimit).filter(x => x.length >= 1 && x.length <= 16);
@@ -2382,7 +2389,7 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov
23822389 state.reasoning += data?.delta?.thinking || '';
23832390 }
23842391 return data?.delta?.text || '';
23852392 } else if (chat_completion_source === [chat_completion_sources.MAKERSUITE, chat_completion_sources.VERTEXAI].includes(chat_completion_source)) {
23862393 const inlineData = data?.candidates?.[0]?.content?.parts?.find(x => x.inlineData)?.inlineData;
23872394 if (inlineData) {
23882395 state.image = `data:${inlineData.mimeType};base64,${inlineData.data}`;
@@ -2772,7 +2779,13 @@ class Message {
27722779 * @returns {Promise<string>} Compressed image as a Data URL.
27732780 */
27742781 async compressImage(image) {
2775- if ([chat_completion_sources.OPENROUTER, chat_completion_sources.MAKERSUITE, chat_completion_sources.MISTRALAI].includes(oai_settings.chat_completion_source)) {
2782+ const compressImageSources = [
2783+ chat_completion_sources.OPENROUTER,
2784+ chat_completion_sources.MAKERSUITE,
2785+ chat_completion_sources.MISTRALAI,
2786+ chat_completion_sources.VERTEXAI,
2787+ ];
2788+ if (compressImageSources.includes(oai_settings.chat_completion_source)) {
27762789 const sizeThreshold = 2 * 1024 * 1024;
27772790 const dataSize = image.length * 0.75;
27782791 const maxSide = 1024;
@@ -3368,6 +3381,7 @@ function loadOpenAISettings(data, settings) {
33683381 oai_settings.custom_include_headers = settings.custom_include_headers ?? default_settings.custom_include_headers;
33693382 oai_settings.custom_prompt_post_processing = settings.custom_prompt_post_processing ?? default_settings.custom_prompt_post_processing;
33703383 oai_settings.google_model = settings.google_model ?? default_settings.google_model;
3384+ oai_settings.vertexai_model = settings.vertexai_model ?? default_settings.vertexai_model;
33713385 oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source;
33723386 oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale;
33733387 oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models;
@@ -3432,6 +3446,8 @@ function loadOpenAISettings(data, settings) {
34323446 $(`#model_windowai_select option[value="${oai_settings.windowai_model}"`).prop('selected', true);
34333447 $('#model_google_select').val(oai_settings.google_model);
34343448 $(`#model_google_select option[value="${oai_settings.google_model}"`).prop('selected', true);
3449+ $('#model_vertexai_select').val(oai_settings.vertexai_model);
3450+ $(`#model_vertexai_select option[value="${oai_settings.vertexai_model}"`).prop('selected', true);
34353451 $('#model_ai21_select').val(oai_settings.ai21_model);
34363452 $(`#model_ai21_select option[value="${oai_settings.ai21_model}"`).prop('selected', true);
34373453 $('#model_mistralai_select').val(oai_settings.mistralai_model);
@@ -3627,6 +3643,7 @@ async function getStatusOpen() {
36273643 chat_completion_sources.CLAUDE,
36283644 chat_completion_sources.AI21,
36293645 chat_completion_sources.MAKERSUITE,
3646+ chat_completion_sources.VERTEXAI,
36303647 chat_completion_sources.PERPLEXITY,
36313648 chat_completion_sources.GROQ,
36323649 ];
@@ -3648,7 +3665,16 @@ async function getStatusOpen() {
36483665 chat_completion_source: oai_settings.chat_completion_source,
36493666 };
36503667
3651- if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI, chat_completion_sources.MISTRALAI, chat_completion_sources.MAKERSUITE, chat_completion_sources.DEEPSEEK, chat_completion_sources.XAI].includes(oai_settings.chat_completion_source)) {
3668+ const validateProxySources = [
3669+ chat_completion_sources.CLAUDE,
3670+ chat_completion_sources.OPENAI,
3671+ chat_completion_sources.MISTRALAI,
3672+ chat_completion_sources.MAKERSUITE,
3673+ chat_completion_sources.VERTEXAI,
3674+ chat_completion_sources.DEEPSEEK,
3675+ chat_completion_sources.XAI,
3676+ ];
3677+ if (oai_settings.reverse_proxy && validateProxySources.includes(oai_settings.chat_completion_source)) {
36523678 await validateReverseProxy();
36533679 }
36543680
@@ -3740,6 +3766,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
37403766 custom_include_headers: settings.custom_include_headers,
37413767 custom_prompt_post_processing: settings.custom_prompt_post_processing,
37423768 google_model: settings.google_model,
3769+ vertexai_model: settings.vertexai_model,
37433770 temperature: settings.temp_openai,
37443771 frequency_penalty: settings.freq_pen_openai,
37453772 presence_penalty: settings.pres_pen_openai,
@@ -4495,6 +4522,11 @@ async function onModelChange() {
44954522 oai_settings.google_model = value;
44964523 }
44974524
4525+ if ($(this).is('#model_vertexai_select')) {
4526+ console.log('Vertex AI model changed to', value);
4527+ oai_settings.vertexai_model = value;
4528+ }
4529+
44984530 if ($(this).is('#model_mistralai_select')) {
44994531 // Upgrade old mistral models to new naming scheme
45004532 // would have done this in loadOpenAISettings, but it wasn't updating on preset change?
@@ -4575,7 +4607,7 @@ async function onModelChange() {
45754607 $('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input');
45764608 }
45774609
45784610 if (oai_settings[chat_completion_sources.chat_completion_source ==MAKERSUITE, chat_completion_sources.MAKERSUITEVERTEXAI].includes(oai_settings.chat_completion_source)) {
45794611 if (oai_settings.max_context_unlocked) {
45804612 $('#openai_max_context').attr('max', max_2mil);
45814613 } else if (value.includes('gemini-1.5-pro')) {
@@ -4943,6 +4975,19 @@ async function onConnectButtonClick(e) {
49434975 }
49444976 }
49454977
4978+ if (oai_settings.chat_completion_source == chat_completion_sources.VERTEXAI) {
4979+ const api_key_vertexai = String($('#api_key_vertexai').val()).trim();
4980+
4981+ if (api_key_vertexai.length) {
4982+ await writeSecret(SECRET_KEYS.VERTEXAI, api_key_vertexai);
4983+ }
4984+
4985+ if (!secret_state[SECRET_KEYS.VERTEXAI] && !oai_settings.reverse_proxy) {
4986+ console.log('No secret key saved for Vertex AI');
4987+ return;
4988+ }
4989+ }
4990+
49464991 if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
49474992 const api_key_claude = String($('#api_key_claude').val()).trim();
49484993
@@ -5120,6 +5165,9 @@ function toggleChatCompletionForms() {
51205165 else if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
51215166 $('#model_google_select').trigger('change');
51225167 }
5168+ else if (oai_settings.chat_completion_source == chat_completion_sources.VERTEXAI) {
5169+ $('#model_vertexai_select').trigger('change');
5170+ }
51235171 else if (oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) {
51245172 $('#model_openrouter_select').trigger('change');
51255173 }
@@ -5281,6 +5329,8 @@ export function isImageInliningSupported() {
52815329 );
52825330 case chat_completion_sources.MAKERSUITE:
52835331 return visionSupportedModels.some(model => oai_settings.google_model.includes(model));
5332+ case chat_completion_sources.VERTEXAI:
5333+ return visionSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
52845334 case chat_completion_sources.CLAUDE:
52855335 return visionSupportedModels.some(model => oai_settings.claude_model.includes(model));
52865336 case chat_completion_sources.OPENROUTER:
@@ -5887,6 +5937,7 @@ export function initOpenAI() {
58875937 $('#model_windowai_select').on('change', onModelChange);
58885938 $('#model_scale_select').on('change', onModelChange);
58895939 $('#model_google_select').on('change', onModelChange);
5940+ $('#model_vertexai_select').on('change', onModelChange);
58905941 $('#model_openrouter_select').on('change', onModelChange);
58915942 $('#openrouter_group_models').on('change', onOpenrouterModelSortChange);
58925943 $('#openrouter_sort_models').on('change', onOpenrouterModelSortChange);
public/scripts/reasoning.js+1 -0
@@ -114,6 +114,7 @@ export function extractReasoningFromData(data, {
114114 case chat_completion_sources.OPENROUTER:
115115 return data?.choices?.[0]?.message?.reasoning ?? '';
116116 case chat_completion_sources.MAKERSUITE:
117+ case chat_completion_sources.VERTEXAI:
117118 return data?.responseContent?.parts?.filter(part => part.thought)?.map(part => part.text)?.join('\n\n') ?? '';
118119 case chat_completion_sources.CLAUDE:
119120 return data?.content?.find(part => part.type === 'thinking')?.thinking ?? '';
public/scripts/secrets.js+2 -0
@@ -16,6 +16,7 @@ export const SECRET_KEYS = {
1616 AI21: 'api_key_ai21',
1717 SCALE_COOKIE: 'scale_cookie',
1818 MAKERSUITE: 'api_key_makersuite',
19+ VERTEXAI: 'api_key_vertexai',
1920 SERPAPI: 'api_key_serpapi',
2021 MISTRALAI: 'api_key_mistralai',
2122 TOGETHERAI: 'api_key_togetherai',
@@ -56,6 +57,7 @@ const INPUT_MAP = {
5657 [SECRET_KEYS.AI21]: '#api_key_ai21',
5758 [SECRET_KEYS.SCALE_COOKIE]: '#scale_cookie',
5859 [SECRET_KEYS.MAKERSUITE]: '#api_key_makersuite',
60+ [SECRET_KEYS.VERTEXAI]: '#api_key_vertexai',
5961 [SECRET_KEYS.VLLM]: '#api_key_vllm',
6062 [SECRET_KEYS.APHRODITE]: '#api_key_aphrodite',
6163 [SECRET_KEYS.TABBY]: '#api_key_tabby',
public/scripts/slash-commands.js+1 -0
@@ -4127,6 +4127,7 @@ function getModelOptions(quiet) {
41274127 { id: 'model_openrouter_select', api: 'openai', type: chat_completion_sources.OPENROUTER },
41284128 { id: 'model_ai21_select', api: 'openai', type: chat_completion_sources.AI21 },
41294129 { id: 'model_google_select', api: 'openai', type: chat_completion_sources.MAKERSUITE },
4130+ { id: 'model_vertexai_select', api: 'openai', type: chat_completion_sources.VERTEXAI },
41304131 { id: 'model_mistralai_select', api: 'openai', type: chat_completion_sources.MISTRALAI },
41314132 { id: 'custom_model_id', api: 'openai', type: chat_completion_sources.CUSTOM },
41324133 { id: 'model_cohere_select', api: 'openai', type: chat_completion_sources.COHERE },
public/scripts/tokenizers.js+4 -0
@@ -676,6 +676,10 @@ export function getTokenizerModel() {
676676 return gemmaTokenizer;
677677 }
678678
679+ if (oai_settings.chat_completion_source == chat_completion_sources.VERTEXAI) {
680+ return gemmaTokenizer;
681+ }
682+
679683 if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
680684 return jambaTokenizer;
681685 }
public/scripts/tool-calling.js+1 -0
@@ -592,6 +592,7 @@ export class ToolManager {
592592 chat_completion_sources.COHERE,
593593 chat_completion_sources.DEEPSEEK,
594594 chat_completion_sources.MAKERSUITE,
595+ chat_completion_sources.VERTEXAI,
595596 chat_completion_sources.AI21,
596597 chat_completion_sources.XAI,
597598 chat_completion_sources.POLLINATIONS,
src/constants.js+1 -0
@@ -168,6 +168,7 @@ export const CHAT_COMPLETION_SOURCES = {
168168 OPENROUTER: 'openrouter',
169169 AI21: 'ai21',
170170 MAKERSUITE: 'makersuite',
171+ VERTEXAI: 'vertexai',
171172 MISTRALAI: 'mistralai',
172173 CUSTOM: 'custom',
173174 COHERE: 'cohere',
src/endpoints/backends/chat-completions.js+33 -10
@@ -52,6 +52,7 @@ const API_COHERE_V2 = 'https://api.cohere.ai/v2';
5252const API_PERPLEXITY = 'https://api.perplexity.ai';
5353const API_GROQ = 'https://api.groq.com/openai/v1';
5454const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
55+const API_VERTEX_AI = 'https://us-central1-aiplatform.googleapis.com';
5556const API_01AI = 'https://api.lingyiwanwu.com/v1';
5657const API_AI21 = 'https://api.ai21.com/studio/v1';
5758const API_NANOGPT = 'https://nano-gpt.com/api/v1';
@@ -337,13 +338,28 @@ async function sendScaleRequest(request, response) {
337338 * @param {express.Response} response Express response
338339 */
339340async function sendMakerSuiteRequest(request, response) {
340341 const apiUrluseVertexAi = new URL(request.body.reverse_proxychat_completion_source ||=== API_MAKERSUITE)CHAT_COMPLETION_SOURCES.VERTEXAI;
341- const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
342+ const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
343+ let apiUrl;
344+ let apiKey;
345+
346+ if (useVertexAi) {
347+ apiUrl = new URL(request.body.reverse_proxy || API_VERTEX_AI);
348+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.VERTEXAI);
342349
343350 if (!request.body.reverse_proxy && !apiKey) {
344351 console.warn('Google AI Studio`${apiName} API key is missing.'`);
345352 return response.status(400).send({ error: true });
346353 }
354+ } else {
355+ apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
356+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
357+
358+ if (!request.body.reverse_proxy && !apiKey) {
359+ console.warn(`${apiName} API key is missing.`);
360+ return response.status(400).send({ error: true });
361+ }
362+ }
347363
348364 const model = String(request.body.model);
349365 const stream = Boolean(request.body.stream);
@@ -465,7 +481,7 @@ async function sendMakerSuiteRequest(request, response) {
465481 }
466482
467483 const body = getGeminiBody();
468484 console.debug('Google AI Studio`${apiName} request:'`, body);
469485
470486 try {
471487 const controller = new AbortController();
@@ -477,7 +493,13 @@ async function sendMakerSuiteRequest(request, response) {
477493 const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
478494 const responseType = (stream ? 'streamGenerateContent' : 'generateContent');
479495
480- const generateResponse = await fetch(`${apiUrl.toString().replace(/\/$/, '')}/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`, {
496+ let url;
497+ if (useVertexAi) {
498+ url = `${apiUrl.toString().replace(/\/$/, '')}/v1/publishers/google/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`;
499+ } else {
500+ url = `${apiUrl.toString().replace(/\/$/, '')}/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`;
501+ }
502+ const generateResponse = await fetch(url, {
481503 body: JSON.stringify(body),
482504 method: 'POST',
483505 headers: {
@@ -498,7 +520,7 @@ async function sendMakerSuiteRequest(request, response) {
498520 }
499521 } else {
500522 if (!generateResponse.ok) {
501523 console.warn(`Google AI Studio${apiName} API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`);
502524 return response.status(500).send({ error: true });
503525 }
504526
@@ -507,7 +529,7 @@ async function sendMakerSuiteRequest(request, response) {
507529
508530 const candidates = generateResponseJson?.candidates;
509531 if (!candidates || candidates.length === 0) {
510532 let message = 'Google AI Studio`${apiName} API returned no candidate'`;
511533 console.warn(message, generateResponseJson);
512534 if (generateResponseJson?.promptFeedback?.blockReason) {
513535 message += `\nPrompt was blocked due to : ${generateResponseJson.promptFeedback.blockReason}`;
@@ -518,11 +540,11 @@ async function sendMakerSuiteRequest(request, response) {
518540 const responseContent = candidates[0].content ?? candidates[0].output;
519541 const functionCall = (candidates?.[0]?.content?.parts ?? []).some(part => part.functionCall);
520542 const inlineData = (candidates?.[0]?.content?.parts ?? []).some(part => part.inlineData);
521543 console.debug('Google AI Studio`${apiName} response:'`, util.inspect(generateResponseJson, { depth: 5, colors: true }));
522544
523545 const responseText = typeof responseContent === 'string' ? responseContent : responseContent?.parts?.filter(part => !part.thought)?.map(part => part.text)?.join('\n\n');
524546 if (!responseText && !functionCall && !inlineData) {
525547 let message = 'Google AI Studio`${apiName} Candidate text empty'`;
526548 console.warn(message, generateResponseJson);
527549 return response.send({ error: { message } });
528550 }
@@ -532,7 +554,7 @@ async function sendMakerSuiteRequest(request, response) {
532554 return response.send(reply);
533555 }
534556 } catch (error) {
535557 console.error('`Error communicating with Google AI Studio${apiName} API: '`, error);
536558 if (!response.headersSent) {
537559 return response.status(500).send({ error: true });
538560 }
@@ -1189,6 +1211,7 @@ router.post('/generate', function (request, response) {
11891211 case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response);
11901212 case CHAT_COMPLETION_SOURCES.AI21: return sendAI21Request(request, response);
11911213 case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response);
1214+ case CHAT_COMPLETION_SOURCES.VERTEXAI: return sendMakerSuiteRequest(request, response);
11921215 case CHAT_COMPLETION_SOURCES.MISTRALAI: return sendMistralAIRequest(request, response);
11931216 case CHAT_COMPLETION_SOURCES.COHERE: return sendCohereRequest(request, response);
11941217 case CHAT_COMPLETION_SOURCES.DEEPSEEK: return sendDeepSeekRequest(request, response);
src/endpoints/google.js+22 -6
@@ -7,6 +7,7 @@ import { readSecret, SECRET_KEYS } from './secrets.js';
77import { GEMINI_SAFETY } from '../constants.js';
88
99const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
10+const API_VERTEX_AI = 'https://us-central1-aiplatform.googleapis.com';
1011
1112export const router = express.Router();
1213
@@ -14,12 +15,27 @@ router.post('/caption-image', async (request, response) => {
1415 try {
1516 const mimeType = request.body.image.split(';')[0].split(':')[1];
1617 const base64Data = request.body.image.split(',')[1];
17- const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
18+ const useVertexAi = request.body.api === 'vertexai';
18- const apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
19+ const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
20+ let apiKey;
21+ let apiUrl;
22+ if (useVertexAi) {
23+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.VERTEXAI);
24+ apiUrl = new URL(request.body.reverse_proxy || API_VERTEX_AI);
25+ } else {
26+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
27+ apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
28+ }
1929 const model = request.body.model || 'gemini-2.0-flash';
20- const url = `${apiUrl.origin}/v1beta/models/${model}:generateContent?key=${apiKey}`;
30+ let url;
31+ if (useVertexAi) {
32+ url = `${apiUrl.origin}/v1/publishers/google/models/${model}:generateContent?key=${apiKey}`;
33+ } else {
34+ url = `${apiUrl.origin}/v1beta/models/${model}:generateContent?key=${apiKey}`;
35+ }
2136 const body = {
2237 contents: [{
38+ role: 'user',
2339 parts: [
2440 { text: request.body.prompt },
2541 {
@@ -32,7 +48,7 @@ router.post('/caption-image', async (request, response) => {
3248 safetySettings: GEMINI_SAFETY,
3349 };
3450
3551 console.debug('Multimodal`${apiName} captioning request'`, model, body);
3652
3753 const result = await fetch(url, {
3854 body: JSON.stringify(body),
@@ -44,13 +60,13 @@ router.post('/caption-image', async (request, response) => {
4460
4561 if (!result.ok) {
4662 const error = await result.json();
4763 console.error(`Google AI Studio${apiName} API returned error: ${result.status} ${result.statusText}`, error);
4864 return response.status(500).send({ error: true });
4965 }
5066
5167 /** @type {any} */
5268 const data = await result.json();
5369 console.info('Multimodal`${apiName} captioning response'`, data);
5470
5571 const candidates = data?.candidates;
5672 if (!candidates) {
src/endpoints/secrets.js+1 -0
@@ -26,6 +26,7 @@ export const SECRET_KEYS = {
2626 ONERING_URL: 'oneringtranslator_url',
2727 DEEPLX_URL: 'deeplx_url',
2828 MAKERSUITE: 'api_key_makersuite',
29+ VERTEXAI: 'api_key_vertexai',
2930 SERPAPI: 'api_key_serpapi',
3031 TOGETHERAI: 'api_key_togetherai',
3132 MISTRALAI: 'api_key_mistralai',
src/endpoints/tokenizers.js+1 -1
@@ -463,7 +463,7 @@ export function getTokenizerModel(requestModel) {
463463 return 'deepseek';
464464 }
465465
466466 if (requestModel.includes('gemma') || requestModel.includes('gemini') || requestModel.includes('learnlm')) {
467467 return 'gemma';
468468 }
469469