feat: add Workers AI text embeddings and multimodal captioning (#5414) * feat: add Workers AI text embeddings and multimodal captioning Extends the Cloudflare Workers AI integration to the vectors and caption extensions. Embeddings: adds workers_ai source to the vectors extension using the OpenAI-compatible /v1/embeddings endpoint, with dynamic model listing from the Cloudflare model search API. Captioning: adds workers_ai as a multimodal caption API with dynamic vision model discovery via the multimodal-models endpoint. * Add logo svg * Refactor caption dropdown population * Fix order of sources * feat: add error handling for missing Workers AI account ID --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

a9c377c3c8e70e48b1f1e1f03b00b9a0f3f67157

Tony Gies <tgies@tgies.net>

Signed
Parents: 94a5773412a479b801d64f252c832771e0f4b3b1
10 files changed, +211 -3Ignore whitespace
public/img/workers_ai.svg+9 -0
@@ -0,0 +1,9 @@
1+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="64" height="64">
2+ <path d="M8.16 23h21.177v-5.86l-4.023-2.307-.694-.3-16.46.113z" fill="none" />
3+ <path
4+ d="M22.012 22.222c.197-.675.122-1.294-.206-1.754-.3-.422-.807-.666-1.416-.694l-11.545-.15c-.075 0-.14-.038-.178-.094s-.047-.13-.028-.206c.038-.113.15-.197.272-.206l11.648-.15c1.38-.066 2.88-1.182 3.404-2.55l.666-1.735a.38.38 0 0 0 .02-.225c-.75-3.395-3.78-5.927-7.4-5.927-3.34 0-6.17 2.157-7.184 5.15-.657-.488-1.5-.75-2.392-.666-1.604.16-2.9 1.444-3.048 3.048a3.58 3.58 0 0 0 .084 1.191A4.84 4.84 0 0 0 0 22.1c0 .234.02.47.047.703.02.113.113.197.225.197H21.58a.29.29 0 0 0 .272-.206l.16-.572z"
5+ />
6+ <path
7+ d="M25.688 14.803l-.32.01c-.075 0-.14.056-.17.13l-.45 1.566c-.197.675-.122 1.294.206 1.754.3.422.807.666 1.416.694l2.457.15c.075 0 .14.038.178.094s.047.14.028.206c-.038.113-.15.197-.272.206l-2.56.15c-1.388.066-2.88 1.182-3.404 2.55l-.188.478c-.038.094.028.188.13.188h8.797a.23.23 0 0 0 .225-.169A6.41 6.41 0 0 0 32 21.106a6.32 6.32 0 0 0-6.312-6.302"
8+ />
9+</svg>
9 \ No newline at end of file
public/scripts/extensions/caption/index.js+6 -2
@@ -3,6 +3,7 @@ import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, rend
33import { appendMediaToMessage, chat_metadata, eventSource, event_types, getRequestHeaders, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
44import { getMessageTimeStamp } from '../../RossAscends-mods.js';
55import { SECRET_KEYS, secret_state } from '../../secrets.js';
6+import { oai_settings } from '../../openai.js';
67import { getMultimodalCaption } from '../shared.js';
78import { textgen_types, textgenerationwebui_settings } from '../../textgen-settings.js';
89import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
@@ -504,6 +505,7 @@ jQuery(async function () {
504505 'chutes': SECRET_KEYS.CHUTES,
505506 'electronhub': SECRET_KEYS.ELECTRONHUB,
506507 'pollinations': SECRET_KEYS.POLLINATIONS,
508+ 'workers_ai': SECRET_KEYS.WORKERS_AI,
507509 };
508510
509511 if (chatCompletionApis[api] && secret_state[chatCompletionApis[api]]) {
@@ -580,7 +582,7 @@ jQuery(async function () {
580582 }
581583
582584 async function addRemoteEndpointModels() {
583585 async function processEndpoint(api, url, additionalParams = {}) {
584586 const dropdown = document.getElementById('caption_multimodal_model');
585587 if (!(dropdown instanceof HTMLSelectElement)) {
586588 return;
@@ -591,7 +593,8 @@ jQuery(async function () {
591593 const options = Array.from(dropdown.options);
592594 const response = await fetch(url, {
593595 method: 'POST',
594596 headers: getRequestHeaders({ omitContentType: true }),
597+ body: JSON.stringify(additionalParams),
595598 });
596599 if (!response.ok) {
597600 return;
@@ -620,6 +623,7 @@ jQuery(async function () {
620623 await processEndpoint('mistral', '/api/backends/chat-completions/multimodal-models/mistral');
621624 await processEndpoint('xai', '/api/backends/chat-completions/multimodal-models/xai');
622625 await processEndpoint('moonshot', '/api/backends/chat-completions/multimodal-models/moonshot');
626+ await processEndpoint('workers_ai', '/api/backends/chat-completions/multimodal-models/workers_ai', { workers_ai_account_id: oai_settings.workers_ai_account_id });
623627 }
624628
625629 await addSettings();
public/scripts/extensions/caption/settings.html+1 -0
@@ -20,6 +20,7 @@
2020 <option value="aimlapi">AI/ML API</option>
2121 <option value="chutes">Chutes</option>
2222 <option value="anthropic">Claude</option>
23+ <option value="workers_ai">Cloudflare Workers AI</option>
2324 <option value="cohere">Cohere</option>
2425 <option value="custom" data-i18n="Custom (OpenAI-compatible)">Custom (OpenAI-compatible)</option>
2526 <option value="electronhub">Electron Hub</option>
public/scripts/extensions/shared.js+8 -0
@@ -126,6 +126,10 @@ export async function getMultimodalCaption(base64Img, prompt) {
126126 requestBody.zai_endpoint = oai_settings.zai_endpoint || ZAI_ENDPOINT.COMMON;
127127 }
128128
129+ if (extension_settings.caption.multimodal_api === 'workers_ai') {
130+ requestBody.workers_ai_account_id = oai_settings.workers_ai_account_id;
131+ }
132+
129133 function getEndpointUrl() {
130134 switch (extension_settings.caption.multimodal_api) {
131135 case 'google':
@@ -283,6 +287,10 @@ function throwIfInvalidModel(useReverseProxy) {
283287 if (multimodalApi === 'pollinations' && !secret_state[SECRET_KEYS.POLLINATIONS]) {
284288 throw new Error('Pollinations API key is not set.');
285289 }
290+
291+ if (multimodalApi === 'workers_ai' && (!secret_state[SECRET_KEYS.WORKERS_AI] || !oai_settings.workers_ai_account_id)) {
292+ throw new Error('Workers AI API key or account ID is not set.');
293+ }
286294}
287295
288296/**
public/scripts/extensions/vectors/index.js+59 -0
@@ -389,6 +389,8 @@ async function synchronizeChat(batchSize = 5) {
389389 return 'Extras API must provide an "embeddings" module.';
390390 case 'webllm_not_supported':
391391 return 'WebLLM extension is not installed or the model is not set.';
392+ case 'account_id_missing':
393+ return 'Workers AI account ID is required. Save it in the "API Connections" panel.';
392394 default:
393395 return 'Check server console for more details';
394396 }
@@ -843,6 +845,10 @@ function getVectorsRequestBody(args = {}) {
843845 body.model = extension_settings.vectors.siliconflow_model;
844846 body.siliconflow_endpoint = oai_settings.siliconflow_endpoint;
845847 break;
848+ case 'workers_ai':
849+ body.model = extension_settings.vectors.workers_ai_model || '@cf/baai/bge-m3';
850+ body.workers_ai_account_id = oai_settings.workers_ai_account_id;
851+ break;
846852 default:
847853 break;
848854 }
@@ -936,6 +942,7 @@ function throwIfSourceInvalid() {
936942 settings.source === 'togetherai' && !secret_state[SECRET_KEYS.TOGETHERAI] ||
937943 settings.source === 'nomicai' && !secret_state[SECRET_KEYS.NOMICAI] ||
938944 settings.source === 'cohere' && !secret_state[SECRET_KEYS.COHERE] ||
945+ settings.source === 'workers_ai' && !secret_state[SECRET_KEYS.WORKERS_AI] ||
939946 settings.source === 'siliconflow' && !secret_state[SECRET_KEYS.SILICONFLOW]) {
940947 throw new Error('Vectors: API key missing', { cause: 'api_key_missing' });
941948 }
@@ -964,6 +971,10 @@ function throwIfSourceInvalid() {
964971 if (settings.source === 'webllm' && (!isWebLlmSupported() || !settings.webllm_model)) {
965972 throw new Error('Vectors: WebLLM is not supported', { cause: 'webllm_not_supported' });
966973 }
974+
975+ if (settings.source === 'workers_ai' && !oai_settings.workers_ai_account_id) {
976+ throw new Error('Vectors: Workers AI account ID missing', { cause: 'account_id_missing' });
977+ }
967978}
968979
969980/**
@@ -1155,6 +1166,7 @@ function toggleSettings() {
11551166 $('#koboldcpp_vectorsModel').toggle(settings.source === 'koboldcpp');
11561167 $('#google_vectorsModel').toggle(settings.source === 'palm' || settings.source === 'vertexai');
11571168 $('#siliconflow_vectorsModel').toggle(settings.source === 'siliconflow');
1169+ $('#workers_ai_vectorsModel').toggle(settings.source === 'workers_ai');
11581170 $('#vector_altEndpointUrl').toggle(vectorApiRequiresUrl.includes(settings.source));
11591171 switch (settings.source) {
11601172 case 'webllm':
@@ -1175,6 +1187,9 @@ function toggleSettings() {
11751187 case 'siliconflow':
11761188 loadSiliconFlowModels();
11771189 break;
1190+ case 'workers_ai':
1191+ loadWorkersAIModels();
1192+ break;
11781193 }
11791194}
11801195
@@ -1362,6 +1377,45 @@ function populateSiliconFlowModelSelect(models) {
13621377 $('#vectors_siliconflow_model').val(settings.siliconflow_model);
13631378}
13641379
1380+async function loadWorkersAIModels() {
1381+ try {
1382+ const response = await fetch('/api/openai/workers-ai/models/embedding', {
1383+ method: 'POST',
1384+ headers: getRequestHeaders(),
1385+ body: JSON.stringify({
1386+ workers_ai_account_id: oai_settings.workers_ai_account_id,
1387+ }),
1388+ });
1389+ if (!response.ok) {
1390+ throw new Error(`HTTP ${response.status}`);
1391+ }
1392+ /** @type {Array<any>} */
1393+ const data = await response.json();
1394+ const models = Array.isArray(data) ? data : [];
1395+ populateWorkersAIModelSelect(models);
1396+ } catch (err) {
1397+ console.warn('Workers AI models fetch failed', err);
1398+ populateWorkersAIModelSelect([]);
1399+ }
1400+}
1401+
1402+function populateWorkersAIModelSelect(models) {
1403+ const select = $('#vectors_workers_ai_model');
1404+ select.empty();
1405+ for (const m of models) {
1406+ const option = document.createElement('option');
1407+ option.value = m.id;
1408+ option.text = m.id;
1409+ select.append(option);
1410+ }
1411+ if (!settings.workers_ai_model && models.length) {
1412+ settings.workers_ai_model = models[0].id;
1413+ Object.assign(extension_settings.vectors, settings);
1414+ saveSettingsDebounced();
1415+ }
1416+ $('#vectors_workers_ai_model').val(settings.workers_ai_model);
1417+}
1418+
13651419/**
13661420 * Executes a function with WebLLM error handling.
13671421 * @param {function(): Promise<T>} func Function to execute
@@ -1785,6 +1839,11 @@ jQuery(async () => {
17851839 Object.assign(extension_settings.vectors, settings);
17861840 saveSettingsDebounced();
17871841 });
1842+ $('#vectors_workers_ai_model').val(settings.workers_ai_model).on('change', () => {
1843+ settings.workers_ai_model = String($('#vectors_workers_ai_model').val());
1844+ Object.assign(extension_settings.vectors, settings);
1845+ saveSettingsDebounced();
1846+ });
17881847 $('#vectors_openrouter_model').val(settings.openrouter_model).on('change', () => {
17891848 settings.openrouter_model = String($('#vectors_openrouter_model').val());
17901849 Object.assign(extension_settings.vectors, settings);
public/scripts/extensions/vectors/settings.html+11 -0
@@ -11,6 +11,7 @@
1111 </label>
1212 <select id="vectors_source" class="text_pole">
1313 <option value="chutes">Chutes</option>
14+ <option value="workers_ai">Cloudflare Workers AI</option>
1415 <option value="cohere">Cohere</option>
1516 <option value="electronhub">Electron Hub</option>
1617 <option value="extras">Extras (deprecated)</option>
@@ -205,6 +206,16 @@
205206 </i>
206207 </div>
207208
209+ <div class="flex-container flexFlowColumn" id="workers_ai_vectorsModel">
210+ <label for="vectors_workers_ai_model" data-i18n="Vectorization Model">
211+ Vectorization Model
212+ </label>
213+ <select id="vectors_workers_ai_model" class="text_pole"></select>
214+ <i data-i18n="Hint: Set your Workers AI API key and Account ID in API Connections.">
215+ Hint: Set your Workers AI API key and Account ID in API Connections.
216+ </i>
217+ </div>
218+
208219 <div class="flex-container marginTopBot5">
209220 <div class="flex-container flex1 flexFlowColumn" title="How many last messages will be matched for relevance.">
210221 <label for="vectors_query">
src/endpoints/backends/chat-completions.js+33 -0
@@ -2724,6 +2724,39 @@ multimodalModels.post('/moonshot', async (req, res) => {
27242724 }
27252725});
27262726
2727+multimodalModels.post('/workers_ai', async (req, res) => {
2728+ try {
2729+ const key = readSecret(req.user.directories, SECRET_KEYS.WORKERS_AI);
2730+ const accountId = String(req.body.workers_ai_account_id || '').trim();
2731+
2732+ if (!key || !accountId) {
2733+ return res.json([]);
2734+ }
2735+
2736+ const apiUrl = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/models/search?task=Text+Generation&per_page=1000`;
2737+ const response = await fetch(apiUrl, {
2738+ method: 'GET',
2739+ headers: { 'Authorization': 'Bearer ' + key },
2740+ });
2741+
2742+ if (!response.ok) {
2743+ return res.json([]);
2744+ }
2745+
2746+ /** @type {any} */
2747+ const data = await response.json();
2748+ const models = Array.isArray(data?.result)
2749+ ? data.result
2750+ .filter(m => Array.isArray(m.properties) && m.properties.some(p => p.property_id === 'vision' && p.value === 'true'))
2751+ .map(m => m.name)
2752+ : [];
2753+ return res.json(models);
2754+ } catch (error) {
2755+ console.error(error);
2756+ return res.sendStatus(500);
2757+ }
2758+});
2759+
27272760router.use('/multimodal-models', multimodalModels);
27282761
27292762router.post('/process', async function (request, response) {
src/endpoints/openai.js+56 -0
@@ -102,6 +102,10 @@ router.post('/caption-image', async (request, response) => {
102102 bodyParams.seed = Math.floor(Math.random() * Math.pow(2, 32));
103103 }
104104
105+ if (request.body.api === 'workers_ai') {
106+ key = readSecret(request.user.directories, SECRET_KEYS.WORKERS_AI);
107+ }
108+
105109 const noKeyTypes = ['custom', 'ooba', 'koboldcpp', 'vllm', 'llamacpp'];
106110 if (!key && !request.body.reverse_proxy && !noKeyTypes.includes(request.body.api)) {
107111 console.warn('No key found for API', request.body.api);
@@ -216,6 +220,14 @@ router.post('/caption-image', async (request, response) => {
216220 }
217221 }
218222
223+ if (request.body.api === 'workers_ai') {
224+ const accountId = String(request.body.workers_ai_account_id || '').trim();
225+ if (!accountId) {
226+ return response.status(400).send({ error: 'Cloudflare Workers AI Account ID is required' });
227+ }
228+ apiUrl = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/v1/chat/completions`;
229+ }
230+
219231 if (['koboldcpp', 'vllm', 'llamacpp', 'ooba'].includes(request.body.api)) {
220232 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
221233 }
@@ -570,6 +582,50 @@ router.post('/siliconflow/models/embedding', async (request, response) => {
570582 }
571583});
572584
585+router.post('/workers-ai/models/embedding', async (request, response) => {
586+ try {
587+ const key = readSecret(request.user.directories, SECRET_KEYS.WORKERS_AI);
588+
589+ if (!key) {
590+ console.warn('No Workers AI key found');
591+ return response.sendStatus(400);
592+ }
593+
594+ const accountId = String(request.body.workers_ai_account_id || '').trim();
595+ if (!accountId) {
596+ console.warn('No Workers AI account ID found');
597+ return response.sendStatus(400);
598+ }
599+
600+ const apiUrl = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/models/search?task=Text+Embeddings&per_page=100`;
601+ const result = await fetch(apiUrl, {
602+ method: 'GET',
603+ headers: {
604+ Authorization: `Bearer ${key}`,
605+ },
606+ });
607+
608+ if (!result.ok) {
609+ const text = await result.text();
610+ console.warn('Workers AI embedding models request failed', result.statusText, text);
611+ return response.status(500).send(text);
612+ }
613+
614+ /** @type {any} */
615+ const data = await result.json();
616+
617+ if (!Array.isArray(data?.result)) {
618+ console.warn('Workers AI embedding models response invalid', data);
619+ return response.sendStatus(500);
620+ }
621+
622+ return response.json(data.result.map(m => ({ ...m, id: m.name })));
623+ } catch (error) {
624+ console.error('Workers AI embedding models fetch failed', error);
625+ response.sendStatus(500);
626+ }
627+});
628+
573629router.post('/generate-image', async (request, response) => {
574630 try {
575631 const key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
src/endpoints/vectors.js+15 -0
@@ -39,6 +39,7 @@ const SOURCES = [
3939 'chutes',
4040 'nanogpt',
4141 'siliconflow',
42+ 'workers_ai',
4243];
4344
4445/**
@@ -88,6 +89,8 @@ async function getVector(source, sourceSettings, text, isQuery, directories) {
8889 return getOpenAIVector(text, source, directories, sourceSettings.model);
8990 case 'siliconflow':
9091 return getOpenAIVector(text, source, directories, sourceSettings.model, sourceSettings.urlOverride);
92+ case 'workers_ai':
93+ return getOpenAIVector(text, source, directories, sourceSettings.model, sourceSettings.urlOverride);
9194 }
9295
9396 throw new Error(`Unknown vector source ${source}`);
@@ -162,6 +165,9 @@ async function getBatchVector(source, sourceSettings, texts, isQuery, directorie
162165 case 'siliconflow':
163166 results.push(...await getOpenAIBatchVector(batch, source, directories, sourceSettings.model, sourceSettings.urlOverride));
164167 break;
168+ case 'workers_ai':
169+ results.push(...await getOpenAIBatchVector(batch, source, directories, sourceSettings.model, sourceSettings.urlOverride));
170+ break;
165171 default:
166172 throw new Error(`Unknown vector source ${source}`);
167173 }
@@ -260,6 +266,15 @@ function getSourceSettings(source, request) {
260266 urlOverride: request.body.siliconflow_endpoint === 'cn'
261267 ? 'https://api.siliconflow.cn/v1' : null,
262268 };
269+ case 'workers_ai': {
270+ const accountId = String(request.body.workers_ai_account_id || '').trim();
271+ return {
272+ model: String(request.body.model || '@cf/baai/bge-m3'),
273+ urlOverride: accountId
274+ ? `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/v1`
275+ : null,
276+ };
277+ }
263278 default:
264279 return {};
265280 }
src/vectors/openai-vectors.js+13 -1
@@ -61,6 +61,13 @@ const SOURCES = {
6161 headers: {},
6262 processBody: () => {},
6363 },
64+ 'workers_ai': {
65+ secretKey: SECRET_KEYS.WORKERS_AI,
66+ url: '', // Constructed at runtime from account ID via urlOverride
67+ model: '@cf/baai/bge-m3',
68+ headers: {},
69+ processBody: () => {},
70+ },
6471};
6572
6673/**
@@ -88,7 +95,12 @@ export async function getOpenAIBatchVector(texts, source, directories, model = '
8895 }
8996
9097 const modelName = model || config.model;
9198 const url = urlOverride || config.url?.replace('{{MODEL}}', modelName);
99+
100+ if (!url) {
101+ throw new Error(`No API URL configured for source ${source}`);
102+ }
103+
92104 const body = {
93105 input: texts,
94106 model: modelName,