1
/* eslint-disable dot-notation */
2
import process from 'node:process';
3
import util from 'node:util';
4
import express from 'express';
5
import fetch from 'node-fetch';
6
import urlJoin from 'url-join';
7

8
import {
9
    AIMLAPI_HEADERS,
10
    AZURE_OPENAI_KEYS,
11
    CHAT_COMPLETION_SOURCES,
12
    GEMINI_SAFETY,
13
    NANOGPT_REASONING_EFFORT_MAP,
14
    OPENAI_FIXED_REASONING_EFFORT,
15
    OPENAI_REASONING_EFFORT_MAP,
16
    OPENAI_REASONING_EFFORT_MODELS,
17
    OPENAI_VERBOSITY_MODELS,
18
    OPENROUTER_HEADERS,
19
    VERTEX_SAFETY,
20
    SILICONFLOW_ENDPOINT,
21
    MINIMAX_ENDPOINT,
22
    ZAI_ENDPOINT,
23
} from '../../constants.js';
24
import {
25
    forwardFetchResponse,
26
    getConfigValue,
27
    tryParse,
28
    uuidv4,
29
    mergeObjectWithYaml,
30
    excludeKeysByYaml,
31
    color,
32
    trimTrailingSlash,
33
    flattenSchema,
34
} from '../../util.js';
35
import {
36
    convertClaudeMessages,
37
    convertGooglePrompt,
38
    convertTextCompletionPrompt,
39
    convertCohereMessages,
40
    convertMistralMessages,
41
    convertAI21Messages,
42
    convertXAIMessages,
43
    cachingAtDepthForOpenRouterClaude,
44
    cachingAtDepthForClaude,
45
    getPromptNames,
46
    calculateClaudeBudgetTokens,
47
    calculateGoogleBudgetTokens,
48
    postProcessPrompt,
49
    PROMPT_PROCESSING_TYPE,
50
    addAssistantPrefix,
51
    embedOpenRouterMedia,
52
    addReasoningContentToToolCalls,
53
    cachingSystemPromptForOpenRouter,
54
    addOpenRouterSignatures,
55
} from '../../prompt-converters.js';
56

57
import { readSecret, SECRET_KEYS } from '../secrets.js';
58
import {
59
    getTokenizerModel,
60
    getSentencepiceTokenizer,
61
    getTiktokenTokenizer,
62
    sentencepieceTokenizers,
63
    TEXT_COMPLETION_MODELS,
64
    webTokenizers,
65
    getWebTokenizer,
66
} from '../tokenizers.js';
67
import { getVertexAIAuth, getProjectIdFromServiceAccount } from '../google.js';
68

69
const API_OPENAI = 'https://api.openai.com/v1';
70
const API_CLAUDE = 'https://api.anthropic.com/v1';
71
const API_MISTRAL = 'https://api.mistral.ai/v1';
72
const API_COHERE_V1 = 'https://api.cohere.ai/v1';
73
const API_COHERE_V2 = 'https://api.cohere.ai/v2';
74
const API_PERPLEXITY = 'https://api.perplexity.ai';
75
const API_GROQ = 'https://api.groq.com/openai/v1';
76
const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
77
const API_VERTEX_AI = 'https://us-central1-aiplatform.googleapis.com';
78
const API_AI21 = 'https://api.ai21.com/studio/v1';
79
const API_CHUTES = 'https://llm.chutes.ai/v1';
80
const API_ELECTRONHUB = 'https://api.electronhub.ai/v1';
81
const API_NANOGPT = 'https://nano-gpt.com/api/v1';
82
const API_DEEPSEEK = 'https://api.deepseek.com/beta';
83
const API_XAI = 'https://api.x.ai/v1';
84
const API_AIMLAPI = 'https://api.aimlapi.com/v1';
85
const API_POLLINATIONS = 'https://gen.pollinations.ai/v1';
86
const API_MOONSHOT = 'https://api.moonshot.ai/v1';
87
const API_FIREWORKS = 'https://api.fireworks.ai/inference/v1';
88
const API_COMETAPI = 'https://api.cometapi.com/v1';
89
const API_ZAI_COMMON = 'https://api.z.ai/api/paas/v4';
90
const API_ZAI_CODING = 'https://api.z.ai/api/coding/paas/v4';
91
const API_SILICONFLOW = 'https://api.siliconflow.com/v1';
92
const API_SILICONFLOW_CN = 'https://api.siliconflow.cn/v1';
93
const API_MINIMAX = 'https://api.minimax.io/v1';
94
const API_MINIMAX_CN = 'https://api.minimaxi.com/v1';
95
const API_OPENROUTER = 'https://openrouter.ai/api/v1';
96
const API_WORKERS_AI = 'https://api.cloudflare.com/client/v4/accounts';
97

98
/**
99
 * Module-scoped Claude caching configuration values.
100
 */
101
const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
102
const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
103
const cachingAtDepth = (() => {
104
    const value = getConfigValue('claude.cachingAtDepth', -1, 'number');
105
    return Number.isInteger(value) && value >= 0 ? value : -1;
106
})();
107
const enableAdaptiveThinking = getConfigValue('claude.enableAdaptiveThinking', true, 'boolean');
108

109
/**
110
 * Cache for cacheable (writing) OpenRouter model IDs.
111
 * @type {string[]}
112
 */
113
const openRouterCacheableModels = [];
114

115
/**
116
 * Checks if an OpenRouter model supports prompt cache writing.
117
 * Uses a cache to avoid repeated API calls.
118
 * @param {string} modelId - The OpenRouter model ID
119
 * @returns {Promise<boolean>} `true` if the model supports writing cache
120
 */
121
async function isOpenRouterModelCacheable(modelId) {
122
    if (openRouterCacheableModels.includes(modelId)) {
123
        return true;
124
    }
125

126
    try {
127
        const response = await fetch(`${API_OPENROUTER}/models`, {
128
            method: 'GET',
129
            headers: { 'Accept': 'application/json' },
130
            signal: AbortSignal.timeout(5000),
131
        });
132

133
        if (!response.ok) {
134
            console.warn(`OpenRouter models API returned ${response.status}: ${response.statusText}`);
135
            return false;
136
        }
137

138
        /** @type {any} */
139
        const data = await response.json();
140

141
        if (!Array.isArray(data?.data)) {
142
            console.warn('OpenRouter API response format unexpected');
143
            return false;
144
        }
145

146
        const model = data.data.find(m => m.id === modelId);
147
        const supportsCache = model?.pricing?.input_cache_write != null;
148

149
        if (supportsCache) {
150
            openRouterCacheableModels.push(modelId);
151
        }
152

153
        return supportsCache;
154
    } catch (error) {
155
        console.warn(`Failed to check OpenRouter cache support for ${modelId}:`, error.message);
156
        return false;
157
    }
158
}
159

160
/**
161
 * Gets OpenRouter transforms based on the request.
162
 * @param {import('express').Request} request Express request
163
 * @returns {string[] | undefined} OpenRouter transforms
164
 */
165
function getOpenRouterTransforms(request) {
166
    switch (request.body.middleout) {
167
        case 'on':
168
            return ['middle-out'];
169
        case 'off':
170
            return [];
171
        case 'auto':
172
            return undefined;
173
    }
174
}
175

176
/**
177
 * Gets OpenRouter plugins based on the request.
178
 * @param {import('express').Request} request
179
 * @returns {any[]} OpenRouter plugins
180
 */
181
function getOpenRouterPlugins(request) {
182
    const plugins = [];
183

184
    if (request.body.enable_web_search) {
185
        plugins.push({ 'id': 'web' });
186
    }
187

188
    return plugins;
189
}
190

191
/**
192
 * Hacky way to use JSON schema only if json_object format is supported.
193
 * @param {object} bodyParams Additional body parameters
194
 * @param {object[]} messages Array of messages
195
 * @param {object} jsonSchema JSON schema object
196
 */
197
function setJsonObjectFormat(bodyParams, messages, jsonSchema) {
198
    bodyParams['response_format'] = {
199
        type: 'json_object',
200
    };
201
    const message = {
202
        role: 'user',
203
        content: `JSON schema for the response:\n${JSON.stringify(jsonSchema.value, null, 4)}`,
204
    };
205
    messages.push(message);
206
}
207

208
/**
209
 * Sends a request to Claude API.
210
 * @param {express.Request} request Express request
211
 * @param {express.Response} response Express response
212
 */
213
async function sendClaudeRequest(request, response) {
214
    const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
215
    const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE, request.body.secret_id);
216
    const divider = '-'.repeat(process.stdout.columns);
217

218
    if (!apiKey) {
219
        console.warn(color.red(`Claude API key is missing.\n${divider}`));
220
        return response.status(400).send({ error: true });
221
    }
222

223
    try {
224
        const controller = new AbortController();
225
        request.socket.removeAllListeners('close');
226
        request.socket.on('close', function () {
227
            controller.abort();
228
        });
229
        const additionalHeaders = {};
230
        const betaHeaders = ['output-128k-2025-02-19', 'context-1m-2025-08-07'];
231
        const useTools = Array.isArray(request.body.tools) && request.body.tools.length > 0;
232
        const useSystemPrompt = Boolean(request.body.use_sysprompt);
233
        const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, getPromptNames(request));
234
        const useThinking = /^claude-(3-7|opus-4|sonnet-4|haiku-4-5|opus-4-5|opus-4-6|sonnet-4-6|opus-4-7)/.test(request.body.model);
235
        const useWebSearch = /^claude-(3-5|3-7|opus-4|sonnet-4|haiku-4-5|opus-4-5|opus-4-6|sonnet-4-6|opus-4-7)/.test(request.body.model) && Boolean(request.body.enable_web_search);
236
        const isLimitedSampling = /^claude-(opus-4-1|sonnet-4-5|haiku-4-5|opus-4-5|opus-4-6|sonnet-4-6)/.test(request.body.model);
237
        const useVerbosity = /^claude-(opus-4-5|opus-4-6|sonnet-4-6|opus-4-7)/.test(request.body.model);
238
        const noPrefillModel = /^claude-(opus-4-6|sonnet-4-6|opus-4-7)/.test(request.body.model);
239
        const isAdaptiveModel = /^claude-(opus-4-7)/.test(request.body.model) || (enableAdaptiveThinking && /^claude-(opus-4-6|sonnet-4-6)/.test(request.body.model));
240
        const noSamplingModel = /^claude-(opus-4-7)/.test(request.body.model);
241
        let fixThinkingPrefill = false;
242
        // Add custom stop sequences
243
        const stopSequences = [];
244
        if (Array.isArray(request.body.stop)) {
245
            stopSequences.push(...request.body.stop);
246
        }
247

248
        const requestBody = {
249
            /** @type {any} */ system: [],
250
            messages: convertedPrompt.messages,
251
            model: request.body.model,
252
            max_tokens: request.body.max_tokens,
253
            stop_sequences: stopSequences,
254
            temperature: request.body.temperature,
255
            top_p: request.body.top_p,
256
            top_k: request.body.top_k,
257
            stream: request.body.stream,
258
        };
259
        if (useSystemPrompt) {
260
            if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
261
                convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1].cache_control = { type: 'ephemeral', ttl: cacheTTL };
262
            }
263

264
            requestBody.system = convertedPrompt.systemPrompt;
265
        } else {
266
            delete requestBody.system;
267
        }
268
        if (useTools) {
269
            betaHeaders.push('tools-2024-05-16');
270
            requestBody.tool_choice = { type: request.body.tool_choice };
271
            requestBody.tools = request.body.tools
272
                .filter(tool => tool.type === 'function')
273
                .map(tool => tool.function)
274
                .map(fn => ({ name: fn.name, description: fn.description, input_schema: flattenSchema(fn.parameters, request.body.chat_completion_source) }));
275

276
            if (enableSystemPromptCache && requestBody.tools.length) {
277
                requestBody.tools[requestBody.tools.length - 1].cache_control = { type: 'ephemeral', ttl: cacheTTL };
278
            }
279
        }
280

281
        // Structured output is a forced tool
282
        if (request.body.json_schema) {
283
            const jsonTool = {
284
                name: request.body.json_schema.name,
285
                description: request.body.json_schema.description || 'Well-formed JSON object',
286
                input_schema: request.body.json_schema.value,
287
            };
288
            requestBody.tools = [...(requestBody.tools || []), jsonTool];
289
            requestBody.tool_choice = { type: 'tool', name: request.body.json_schema.name };
290
        }
291

292
        if (useWebSearch) {
293
            const webSearchTool = [{
294
                'type': 'web_search_20250305',
295
                'name': 'web_search',
296
            }];
297
            requestBody.tools = [...webSearchTool, ...(requestBody.tools || [])];
298
        }
299

300
        if (cachingAtDepth !== -1) {
301
            cachingAtDepthForClaude(convertedPrompt.messages, cachingAtDepth, cacheTTL);
302
        }
303

304
        if (enableSystemPromptCache || cachingAtDepth !== -1) {
305
            betaHeaders.push('prompt-caching-2024-07-31');
306
            betaHeaders.push('extended-cache-ttl-2025-04-11');
307
        }
308

309
        if (isLimitedSampling) {
310
            if (requestBody.top_p < 1) {
311
                delete requestBody.temperature;
312
            } else {
313
                delete requestBody.top_p;
314
            }
315
        }
316

317
        if (noSamplingModel) {
318
            delete requestBody.temperature;
319
            delete requestBody.top_p;
320
            delete requestBody.top_k;
321
        }
322

323
        const reasoningEffort = request.body.reasoning_effort;
324
        const budgetTokens = calculateClaudeBudgetTokens(requestBody.max_tokens, reasoningEffort, requestBody.stream, isAdaptiveModel);
325

326
        // Adaptive thinking: returns a string effort level (like Gemini 3)
327
        if (useThinking && typeof budgetTokens === 'string') {
328
            fixThinkingPrefill = true;
329
            requestBody.thinking = { type: 'adaptive' };
330
            const includeReasoning = Boolean(request.body.include_reasoning);
331
            if (noSamplingModel && includeReasoning) {
332
                requestBody.thinking.display = 'summarized';
333
            }
334
            requestBody.output_config ??= {};
335
            requestBody.output_config.effort = budgetTokens;
336
            // top_k is not allowed in adaptive mode
337
            delete requestBody.top_k;
338
        } else if (useThinking && Number.isInteger(budgetTokens)) {
339
            // Traditional thinking: returns a numeric budget
340
            fixThinkingPrefill = true;
341
            const minThinkTokens = 1024;
342
            if (requestBody.max_tokens <= minThinkTokens) {
343
                const newValue = requestBody.max_tokens + minThinkTokens;
344
                console.warn(color.yellow(`Claude thinking requires a minimum of ${minThinkTokens} response tokens.`));
345
                console.info(color.blue(`Increasing response length to ${newValue}.`));
346
                requestBody.max_tokens = newValue;
347
            }
348
            requestBody.thinking = {
349
                type: 'enabled',
350
                budget_tokens: budgetTokens,
351
            };
352

353
            // NO I CAN'T SILENTLY IGNORE THE TEMPERATURE.
354
            delete requestBody.temperature;
355
            delete requestBody.top_p;
356
            delete requestBody.top_k;
357
        }
358

359
        if ((fixThinkingPrefill || noPrefillModel) && convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {
360
            convertedPrompt.messages[convertedPrompt.messages.length - 1].role = 'user';
361
        }
362

363
        // Verbosity = 'effort' (same values as OpenAI) - only if not already set by adaptive thinking
364
        if (useVerbosity && request.body.verbosity && !requestBody.output_config?.effort) {
365
            betaHeaders.push('effort-2025-11-24');
366
            requestBody.output_config ??= {};
367
            requestBody.output_config.effort = request.body.verbosity;
368
        }
369

370
        if (betaHeaders.length) {
371
            additionalHeaders['anthropic-beta'] = betaHeaders.join(',');
372
        }
373

374
        console.debug('Claude request:', requestBody);
375

376
        const generateResponse = await fetch(apiUrl + '/messages', {
377
            method: 'POST',
378
            signal: controller.signal,
379
            body: JSON.stringify(requestBody),
380
            headers: {
381
                'Content-Type': 'application/json',
382
                'anthropic-version': '2023-06-01',
383
                'x-api-key': apiKey,
384
                ...additionalHeaders,
385
            },
386
        });
387

388
        if (request.body.stream) {
389
            // Pipe remote SSE stream to Express response
390
            await forwardFetchResponse(generateResponse, response);
391
        } else {
392
            if (!generateResponse.ok) {
393
                const generateResponseText = await generateResponse.text();
394
                console.warn(color.red(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText}\n${generateResponseText}\n${divider}`));
395
                return response.status(500).send({ error: true });
396
            }
397

398
            /** @type {any} */
399
            const generateResponseJson = await generateResponse.json();
400
            const responseText = generateResponseJson?.content?.[0]?.text || '';
401
            console.debug('Claude response:', generateResponseJson);
402

403
            // Wrap it back to OAI format + save the original content
404
            const reply = { choices: [{ 'message': { 'content': responseText } }], content: generateResponseJson.content };
405
            return response.send(reply);
406
        }
407
    } catch (error) {
408
        console.error(color.red(`Error communicating with Claude: ${error}\n${divider}`));
409
        if (!response.headersSent) {
410
            return response.status(500).send({ error: true });
411
        }
412
    }
413
}
414

415
/**
416
 * Sends a request to Google AI API.
417
 * @param {express.Request} request Express request
418
 * @param {express.Response} response Express response
419
 */
420
async function sendMakerSuiteRequest(request, response) {
421
    const useVertexAi = request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.VERTEXAI;
422
    const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio';
423
    let apiUrl;
424
    let apiKey;
425

426
    let authHeader;
427
    let authType;
428

429
    if (useVertexAi) {
430
        apiUrl = new URL(request.body.reverse_proxy || API_VERTEX_AI);
431

432
        try {
433
            const auth = await getVertexAIAuth(request);
434
            authHeader = auth.authHeader;
435
            authType = auth.authType;
436
            console.debug(`Using Vertex AI authentication type: ${authType}`);
437
        } catch (error) {
438
            console.warn(`${apiName} authentication failed: ${error.message}`);
439
            return response.status(400).send({ error: true, message: error.message });
440
        }
441
    } else {
442
        apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
443
        apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE, request.body.secret_id);
444

445
        if (!request.body.reverse_proxy && !apiKey) {
446
            console.warn(`${apiName} API key is missing.`);
447
            return response.status(400).send({ error: true });
448
        }
449

450
        authHeader = `Bearer ${apiKey}`;
451
        authType = 'api_key';
452
    }
453

454
    const model = String(request.body.model);
455
    const stream = Boolean(request.body.stream);
456
    const enableWebSearch = Boolean(request.body.enable_web_search);
457
    const requestImages = Boolean(request.body.request_images);
458
    const reasoningEffort = String(request.body.reasoning_effort);
459
    const includeReasoning = Boolean(request.body.include_reasoning);
460
    const aspectRatio = String(request.body.request_image_aspect_ratio);
461
    const imageSize = String(request.body.request_image_resolution);
462
    const isGemma3 = /gemma-3/.test(model);
463
    const isLearnLM = model.includes('learnlm');
464

465
    const responseMimeType = request.body.responseMimeType ?? (request.body.json_schema ? 'application/json' : undefined);
466
    const responseSchema = request.body.responseSchema ?? (request.body.json_schema ? request.body.json_schema.value : undefined);
467

468
    const generationConfig = {
469
        stopSequences: request.body.stop,
470
        candidateCount: 1,
471
        maxOutputTokens: request.body.max_tokens,
472
        temperature: request.body.temperature,
473
        topP: request.body.top_p,
474
        topK: request.body.top_k || undefined,
475
        responseMimeType: responseMimeType,
476
        responseSchema: responseSchema,
477
        seed: request.body.seed,
478
    };
479

480
    function getGeminiBody() {
481
        // #region UGLY MODEL LISTS AREA
482
        const imageGenerationModels = [
483
            'gemini-2.0-flash-exp',
484
            'gemini-2.0-flash-exp-image-generation',
485
            'gemini-2.0-flash-preview-image-generation',
486
            'gemini-2.5-flash-image-preview',
487
            'gemini-2.5-flash-image',
488
            'gemini-3-pro-image-preview',
489
            'gemini-3.1-flash-image-preview',
490
        ];
491

492
        const isThinkingConfigModel = m => (/^gemini-2.5-(flash|pro)/.test(m) && !/-image(-preview)?$/.test(m)) || (/^gemini-3[.\d]*-(flash|pro)/.test(m));
493
        const isImageSizeModel = m => /^gemini-3/.test(m);
494

495
        const noSearchModels = [
496
            'gemini-2.0-flash-lite',
497
            'gemini-2.0-flash-lite-001',
498
            'gemini-2.0-flash-lite-preview-02-05',
499
            'gemini-robotics-er-1.5-preview',
500
        ];
501
        // #endregion
502

503
        if (!Array.isArray(generationConfig.stopSequences) || !generationConfig.stopSequences.length) {
504
            delete generationConfig.stopSequences;
505
        }
506

507
        const enableImageModality = requestImages && imageGenerationModels.includes(model);
508
        const enableImageConfig = enableImageModality && (aspectRatio || imageSize);
509
        if (enableImageModality) {
510
            generationConfig.responseModalities = ['text', 'image'];
511
            if (enableImageConfig) {
512
                generationConfig.imageConfig = {};
513
                if (imageSize && isImageSizeModel(model)) {
514
                    generationConfig.imageConfig.imageSize = imageSize;
515
                }
516
                if (aspectRatio) {
517
                    generationConfig.imageConfig.aspectRatio = aspectRatio;
518
                }
519
            }
520
        }
521

522
        const useSystemPrompt = !enableImageModality && !isGemma3 && request.body.use_sysprompt;
523

524
        const tools = [];
525
        const prompt = convertGooglePrompt(request.body.messages, model, useSystemPrompt, getPromptNames(request));
526
        const safetySettings = [...GEMINI_SAFETY, ...(useVertexAi ? VERTEX_SAFETY : [])];
527

528
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0 && !enableImageModality && !isGemma3) {
529
            const functionDeclarations = [];
530
            const customTools = [];
531
            for (const tool of request.body.tools) {
532
                if (tool.type === 'function') {
533
                    if (tool.function.parameters?.$schema) {
534
                        delete tool.function.parameters.$schema;
535
                    }
536
                    if (tool.function.parameters?.properties && Object.keys(tool.function.parameters.properties).length === 0) {
537
                        delete tool.function.parameters;
538
                    }
539
                    functionDeclarations.push(tool.function);
540
                } else if (tool[tool.type]) {
541
                    customTools.push({ [tool.type]: tool[tool.type] });
542
                }
543
            }
544
            if (functionDeclarations.length > 0) {
545
                tools.push({ function_declarations: functionDeclarations });
546
            }
547
            // Custom tools are only supported when no function calling is present
548
            if (functionDeclarations.length === 0 && customTools.length > 0) {
549
                tools.push(...customTools);
550
            }
551
        }
552

553
        if (enableWebSearch && !enableImageModality && !isGemma3 && !isLearnLM && !noSearchModels.includes(model)) {
554
            // Tool use with function calling is unsupported
555
            if (!tools.some(t => t.function_declarations)) {
556
                tools.push({ google_search: {} });
557
            }
558
        }
559

560
        if (isThinkingConfigModel(model)) {
561
            const thinkingConfig = { includeThoughts: includeReasoning };
562

563
            const thinkingBudget = calculateGoogleBudgetTokens(generationConfig.maxOutputTokens, reasoningEffort, model);
564
            if (typeof thinkingBudget === 'number' && Number.isInteger(thinkingBudget)) {
565
                thinkingConfig.thinkingBudget = thinkingBudget;
566
            }
567

568
            if (typeof thinkingBudget === 'string' && thinkingBudget.length > 0) {
569
                thinkingConfig.thinkingLevel = thinkingBudget;
570
            }
571

572
            // Vertex doesn't allow mixing disabled thinking with includeThoughts
573
            if (useVertexAi && thinkingBudget === 0 && thinkingConfig.includeThoughts) {
574
                console.info('Thinking budget is 0, but includeThoughts is true. Thoughts will not be included in the response.');
575
                thinkingConfig.includeThoughts = false;
576
            }
577

578
            generationConfig.thinkingConfig = thinkingConfig;
579
        }
580

581
        let body = {
582
            contents: prompt.contents,
583
            safetySettings: safetySettings,
584
            generationConfig: generationConfig,
585
        };
586

587
        if (useSystemPrompt && Array.isArray(prompt.system_instruction.parts) && prompt.system_instruction.parts.length) {
588
            body.systemInstruction = prompt.system_instruction;
589
        }
590

591
        if (tools.length) {
592
            body.tools = tools;
593

594
            const toolChoice = request.body.tool_choice;
595
            let functionCallingConfig;
596

597
            // Translate OpenAI's `tool_choice` to Gemini's `functionCallingConfig`
598
            if (typeof toolChoice === 'string') {
599
                switch (toolChoice) {
600
                    case 'none':
601
                        functionCallingConfig = { mode: 'NONE' };
602
                        break;
603
                    case 'required':
604
                        functionCallingConfig = { mode: 'ANY' };
605
                        break;
606
                    case 'auto':
607
                        functionCallingConfig = { mode: 'AUTO' };
608
                        break;
609
                }
610
            } else if (typeof toolChoice === 'object' && toolChoice?.function?.name) {
611
                // Force a specific function call
612
                functionCallingConfig = {
613
                    mode: 'ANY',
614
                    allowedFunctionNames: [toolChoice.function.name],
615
                };
616
            }
617

618
            if (functionCallingConfig) {
619
                body.toolConfig = { functionCallingConfig };
620
            }
621
        }
622

623
        return body;
624
    }
625

626
    const body = getGeminiBody();
627
    console.debug(`${apiName} request:`, body);
628

629
    try {
630
        const controller = new AbortController();
631
        request.socket.removeAllListeners('close');
632
        request.socket.on('close', function () {
633
            controller.abort();
634
        });
635

636
        const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
637
        const responseType = (stream ? 'streamGenerateContent' : 'generateContent');
638

639
        let url;
640
        let headers = {
641
            'Content-Type': 'application/json',
642
        };
643

644
        if (useVertexAi) {
645
            if (authType === 'express') {
646
                // For Express mode (API key authentication), use the key parameter
647
                const keyParam = authHeader.replace('Bearer ', '');
648
                const region = request.body.vertexai_region || 'us-central1';
649
                const projectId = request.body.vertexai_express_project_id;
650
                const baseUrl = region === 'global'
651
                    ? 'https://aiplatform.googleapis.com'
652
                    : `https://${region}-aiplatform.googleapis.com`;
653
                url = projectId
654
                    ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${responseType}?key=${keyParam}${stream ? '&alt=sse' : ''}`
655
                    : `${baseUrl}/v1/publishers/google/models/${model}:${responseType}?key=${keyParam}${stream ? '&alt=sse' : ''}`;
656
            } else if (authType === 'full') {
657
                // For Full mode (service account authentication), use project-specific URL
658
                // Get project ID from Service Account JSON
659
                const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT, request.body.secret_id);
660
                if (!serviceAccountJson) {
661
                    console.warn('Vertex AI Service Account JSON is missing.');
662
                    return response.status(400).send({ error: true });
663
                }
664

665
                let projectId;
666
                try {
667
                    const serviceAccount = JSON.parse(serviceAccountJson);
668
                    projectId = getProjectIdFromServiceAccount(serviceAccount);
669
                } catch (error) {
670
                    console.error('Failed to extract project ID from Service Account JSON:', error);
671
                    return response.status(400).send({ error: true });
672
                }
673
                const region = request.body.vertexai_region || 'us-central1';
674
                // Handle global region differently - no region prefix in hostname
675
                if (region === 'global') {
676
                    url = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${responseType}${stream ? '?alt=sse' : ''}`;
677
                } else {
678
                    url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${responseType}${stream ? '?alt=sse' : ''}`;
679
                }
680
                headers['Authorization'] = authHeader;
681
            } else {
682
                // For proxy mode, use the original URL with Authorization header
683
                url = `${apiUrl.toString().replace(/\/$/, '')}/v1/publishers/google/models/${model}:${responseType}${stream ? '?alt=sse' : ''}`;
684
                headers['Authorization'] = authHeader;
685
            }
686
        } else {
687
            url = `${apiUrl.toString().replace(/\/$/, '')}/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`;
688
        }
689

690
        const generateResponse = await fetch(url, {
691
            body: JSON.stringify(body),
692
            method: 'POST',
693
            headers: headers,
694
            signal: controller.signal,
695
        });
696

697
        if (stream) {
698
            try {
699
                // Pipe remote SSE stream to Express response
700
                await forwardFetchResponse(generateResponse, response);
701
            } catch (error) {
702
                console.error('Error forwarding streaming response:', error);
703
                if (!response.headersSent) {
704
                    return response.status(500).send({ error: true });
705
                }
706
            }
707
        } else {
708
            if (!generateResponse.ok) {
709
                const errorText = await generateResponse.text();
710
                console.warn(`${apiName} API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
711
                const errorJson = tryParse(errorText) ?? { error: true };
712
                return response.status(500).send(errorJson);
713
            }
714

715
            /** @type {any} */
716
            const generateResponseJson = await generateResponse.json();
717

718
            const candidates = generateResponseJson?.candidates;
719
            if (!candidates || candidates.length === 0) {
720
                let message = `${apiName} API returned no candidate`;
721
                console.warn(message, generateResponseJson);
722
                if (generateResponseJson?.promptFeedback?.blockReason) {
723
                    message += `\nPrompt was blocked due to : ${generateResponseJson.promptFeedback.blockReason}`;
724
                }
725
                return response.send({ error: { message } });
726
            }
727

728
            const responseContent = candidates[0].content ?? candidates[0].output;
729
            const functionCall = (candidates?.[0]?.content?.parts ?? []).some(part => part.functionCall);
730
            const inlineData = (candidates?.[0]?.content?.parts ?? []).some(part => part.inlineData);
731
            console.debug(`${apiName} response:`, util.inspect(generateResponseJson, { depth: 5, colors: true }));
732

733
            const responseText = typeof responseContent === 'string' ? responseContent : responseContent?.parts?.filter(part => !part.thought)?.map(part => part.text)?.join('\n\n');
734
            if (!responseText && !functionCall && !inlineData) {
735
                let message = `${apiName} Candidate text empty`;
736
                console.warn(message, generateResponseJson);
737
                return response.send({ error: { message } });
738
            }
739

740
            // Wrap it back to OAI format (responseContent includes thought signatures in parts array)
741
            const reply = { choices: [{ 'message': { 'content': responseText } }], responseContent };
742
            return response.send(reply);
743
        }
744
    } catch (error) {
745
        console.error(`Error communicating with ${apiName} API:`, error);
746
        if (!response.headersSent) {
747
            return response.status(500).send({ error: true });
748
        }
749
    }
750
}
751

752
/**
753
 * Sends a request to AI21 API.
754
 * @param {express.Request} request Express request
755
 * @param {express.Response} response Express response
756
 */
757
async function sendAI21Request(request, response) {
758
    if (!request.body) return response.sendStatus(400);
759

760
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.AI21, request.body.secret_id);
761
    if (!apiKey) {
762
        console.warn('AI21 API key is missing.');
763
        return response.status(400).send({ error: true });
764
    }
765

766
    const bodyParams = {};
767
    const controller = new AbortController();
768
    request.socket.removeAllListeners('close');
769
    request.socket.on('close', function () {
770
        controller.abort();
771
    });
772
    // Hack to support JSON schema
773
    if (request.body.json_schema) {
774
        bodyParams.response_format = {
775
            type: 'json_object',
776
        };
777
        const message = {
778
            role: 'user',
779
            content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
780
        };
781
        request.body.messages.push(message);
782
    }
783
    const convertedPrompt = convertAI21Messages(request.body.messages, getPromptNames(request));
784
    const body = {
785
        messages: convertedPrompt,
786
        model: request.body.model,
787
        max_tokens: request.body.max_tokens,
788
        temperature: request.body.temperature,
789
        top_p: request.body.top_p,
790
        stop: request.body.stop,
791
        stream: request.body.stream,
792
        tools: request.body.tools,
793
        ...bodyParams,
794
    };
795
    const options = {
796
        method: 'POST',
797
        headers: {
798
            accept: 'application/json',
799
            'content-type': 'application/json',
800
            Authorization: `Bearer ${apiKey}`,
801
        },
802
        body: JSON.stringify(body),
803
        signal: controller.signal,
804
    };
805

806
    console.debug('AI21 request:', body);
807

808
    try {
809
        const generateResponse = await fetch(API_AI21 + '/chat/completions', options);
810
        if (request.body.stream) {
811
            await forwardFetchResponse(generateResponse, response);
812
        } else {
813
            if (!generateResponse.ok) {
814
                const errorText = await generateResponse.text();
815
                console.warn(`AI21 API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
816
                const errorJson = tryParse(errorText) ?? { error: true };
817
                return response.status(500).send(errorJson);
818
            }
819
            const generateResponseJson = await generateResponse.json();
820
            console.debug('AI21 response:', generateResponseJson);
821
            return response.send(generateResponseJson);
822
        }
823
    } catch (error) {
824
        console.error('Error communicating with AI21 API: ', error);
825
        if (!response.headersSent) {
826
            response.send({ error: true });
827
        } else {
828
            response.end();
829
        }
830
    }
831
}
832

833
/**
834
 * Sends a request to MistralAI API.
835
 * @param {express.Request} request Express request
836
 * @param {express.Response} response Express response
837
 */
838
async function sendMistralAIRequest(request, response) {
839
    const apiUrl = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
840
    const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI, request.body.secret_id);
841

842
    if (!apiKey) {
843
        console.warn('MistralAI API key is missing.');
844
        return response.status(400).send({ error: true });
845
    }
846

847
    try {
848
        const messages = convertMistralMessages(request.body.messages, getPromptNames(request));
849
        const controller = new AbortController();
850
        request.socket.removeAllListeners('close');
851
        request.socket.on('close', function () {
852
            controller.abort();
853
        });
854

855
        const requestBody = {
856
            'model': request.body.model,
857
            'messages': messages,
858
            'temperature': request.body.temperature,
859
            'top_p': request.body.top_p,
860
            'frequency_penalty': request.body.frequency_penalty,
861
            'presence_penalty': request.body.presence_penalty,
862
            'max_tokens': request.body.max_tokens,
863
            'stream': request.body.stream,
864
            'safe_prompt': request.body.safe_prompt,
865
            'random_seed': request.body.seed === -1 ? undefined : request.body.seed,
866
            'stop': Array.isArray(request.body.stop) && request.body.stop.length > 0 ? request.body.stop : undefined,
867
        };
868

869
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
870
            requestBody['tools'] = request.body.tools;
871
            requestBody['tool_choice'] = request.body.tool_choice;
872
        }
873

874
        if (request.body.json_schema) {
875
            requestBody['response_format'] = {
876
                type: 'json_schema',
877
                json_schema: {
878
                    name: request.body.json_schema.name,
879
                    description: request.body.json_schema.description,
880
                    schema: request.body.json_schema.value,
881
                    strict: request.body.json_schema.strict ?? true,
882
                },
883
            };
884
        }
885

886
        const config = {
887
            method: 'POST',
888
            headers: {
889
                'Content-Type': 'application/json',
890
                'Authorization': 'Bearer ' + apiKey,
891
            },
892
            body: JSON.stringify(requestBody),
893
            signal: controller.signal,
894
            timeout: 0,
895
        };
896

897
        console.debug('MisralAI request:', requestBody);
898

899
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
900
        if (request.body.stream) {
901
            await forwardFetchResponse(generateResponse, response);
902
        } else {
903
            if (!generateResponse.ok) {
904
                const errorText = await generateResponse.text();
905
                console.warn(`MistralAI API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
906
                const errorJson = tryParse(errorText) ?? { error: true };
907
                return response.status(500).send(errorJson);
908
            }
909
            const generateResponseJson = await generateResponse.json();
910
            console.debug('MistralAI response:', generateResponseJson);
911
            return response.send(generateResponseJson);
912
        }
913
    } catch (error) {
914
        console.error('Error communicating with MistralAI API: ', error);
915
        if (!response.headersSent) {
916
            response.send({ error: true });
917
        } else {
918
            response.end();
919
        }
920
    }
921
}
922

923
/**
924
 * Sends a request to Cohere API.
925
 * @param {express.Request} request Express request
926
 * @param {express.Response} response Express response
927
 */
928
async function sendCohereRequest(request, response) {
929
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE, request.body.secret_id);
930
    const controller = new AbortController();
931
    request.socket.removeAllListeners('close');
932
    request.socket.on('close', function () {
933
        controller.abort();
934
    });
935

936
    if (!apiKey) {
937
        console.warn('Cohere API key is missing.');
938
        return response.status(400).send({ error: true });
939
    }
940

941
    try {
942
        const convertedHistory = convertCohereMessages(request.body.messages, getPromptNames(request));
943
        const tools = [];
944

945
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
946
            tools.push(...request.body.tools);
947
            tools.forEach(tool => {
948
                if (tool?.function?.parameters?.$schema) {
949
                    delete tool.function.parameters.$schema;
950
                }
951
            });
952
        }
953

954
        // https://docs.cohere.com/reference/chat
955
        const requestBody = {
956
            stream: Boolean(request.body.stream),
957
            model: request.body.model,
958
            messages: convertedHistory.chatHistory,
959
            temperature: request.body.temperature,
960
            max_tokens: request.body.max_tokens,
961
            k: request.body.top_k,
962
            p: request.body.top_p,
963
            seed: request.body.seed,
964
            stop_sequences: request.body.stop,
965
            frequency_penalty: request.body.frequency_penalty,
966
            presence_penalty: request.body.presence_penalty,
967
            documents: [],
968
            tools: tools,
969
        };
970

971
        const canDoSafetyMode = String(request.body.model).endsWith('08-2024');
972
        if (canDoSafetyMode) {
973
            requestBody.safety_mode = 'OFF';
974
        }
975

976
        if (request.body.json_schema) {
977
            requestBody.response_format = {
978
                type: 'json_schema',
979
                schema: request.body.json_schema.value,
980
            };
981
        }
982

983
        console.debug('Cohere request:', requestBody);
984

985
        const config = {
986
            method: 'POST',
987
            headers: {
988
                'Content-Type': 'application/json',
989
                'Authorization': 'Bearer ' + apiKey,
990
            },
991
            body: JSON.stringify(requestBody),
992
            signal: controller.signal,
993
            timeout: 0,
994
        };
995

996
        const apiUrl = API_COHERE_V2 + '/chat';
997

998
        if (request.body.stream) {
999
            const stream = await fetch(apiUrl, config);
1000
            await forwardFetchResponse(stream, response);
1001
        } else {
1002
            const generateResponse = await fetch(apiUrl, config);
1003
            if (!generateResponse.ok) {
1004
                const errorText = await generateResponse.text();
1005
                console.warn(`Cohere API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
1006
                const errorJson = tryParse(errorText) ?? { error: true };
1007
                return response.status(500).send(errorJson);
1008
            }
1009
            const generateResponseJson = await generateResponse.json();
1010
            console.debug('Cohere response:', generateResponseJson);
1011
            return response.send(generateResponseJson);
1012
        }
1013
    } catch (error) {
1014
        console.error('Error communicating with Cohere API: ', error);
1015
        if (!response.headersSent) {
1016
            response.send({ error: true });
1017
        } else {
1018
            response.end();
1019
        }
1020
    }
1021
}
1022

1023
/**
1024
 * Sends a request to DeepSeek API.
1025
 * @param {express.Request} request Express request
1026
 * @param {express.Response} response Express response
1027
 */
1028
async function sendDeepSeekRequest(request, response) {
1029
    const apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK).toString();
1030
    const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK, request.body.secret_id);
1031

1032
    if (!apiKey && !request.body.reverse_proxy) {
1033
        console.warn('DeepSeek API key is missing.');
1034
        return response.status(400).send({ error: true });
1035
    }
1036

1037
    const controller = new AbortController();
1038
    request.socket.removeAllListeners('close');
1039
    request.socket.on('close', function () {
1040
        controller.abort();
1041
    });
1042

1043
    try {
1044
        let bodyParams = {};
1045

1046
        if (request.body.logprobs > 0) {
1047
            bodyParams['top_logprobs'] = request.body.logprobs;
1048
            bodyParams['logprobs'] = true;
1049
        }
1050

1051
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
1052
            bodyParams['tools'] = request.body.tools;
1053
            bodyParams['tool_choice'] = request.body.tool_choice;
1054

1055
            // DeepSeek doesn't permit empty required arrays
1056
            bodyParams.tools.forEach(tool => {
1057
                const required = tool?.function?.parameters?.required;
1058
                if (Array.isArray(required) && required.length === 0) {
1059
                    delete tool.function.parameters.required;
1060
                }
1061
            });
1062
        }
1063

1064
        // Hack to support JSON schema
1065
        if (request.body.json_schema) {
1066
            bodyParams.response_format = {
1067
                type: 'json_object',
1068
            };
1069
            const message = {
1070
                role: 'user',
1071
                content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
1072
            };
1073
            request.body.messages.push(message);
1074
        }
1075

1076
        const processedMessages = addAssistantPrefix(postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.SEMI_TOOLS, getPromptNames(request)), bodyParams.tools, 'prefix');
1077
        addReasoningContentToToolCalls(processedMessages);
1078

1079
        if (request.body.include_reasoning && request.body.reasoning_effort) {
1080
            bodyParams['reasoning_effort'] = request.body.reasoning_effort;
1081
        }
1082

1083
        const requestBody = {
1084
            'messages': processedMessages,
1085
            'model': request.body.model,
1086
            'temperature': request.body.temperature,
1087
            'max_tokens': request.body.max_tokens,
1088
            'stream': request.body.stream,
1089
            'presence_penalty': request.body.presence_penalty,
1090
            'frequency_penalty': request.body.frequency_penalty,
1091
            'top_p': request.body.top_p,
1092
            'stop': request.body.stop,
1093
            'seed': request.body.seed,
1094
            'thinking': { type: request.body.include_reasoning ? 'enabled' : 'disabled' },
1095
            ...bodyParams,
1096
        };
1097

1098
        const config = {
1099
            method: 'POST',
1100
            headers: {
1101
                'Content-Type': 'application/json',
1102
                'Authorization': 'Bearer ' + apiKey,
1103
            },
1104
            body: JSON.stringify(requestBody),
1105
            signal: controller.signal,
1106
        };
1107

1108
        console.debug('DeepSeek request:', requestBody);
1109

1110
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
1111

1112
        if (request.body.stream) {
1113
            await forwardFetchResponse(generateResponse, response);
1114
        } else {
1115
            if (!generateResponse.ok) {
1116
                const errorText = await generateResponse.text();
1117
                console.warn(`DeepSeek API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
1118
                const errorJson = tryParse(errorText) ?? { error: true };
1119
                return response.status(500).send(errorJson);
1120
            }
1121
            const generateResponseJson = await generateResponse.json();
1122
            console.debug('DeepSeek response:', generateResponseJson);
1123
            return response.send(generateResponseJson);
1124
        }
1125
    } catch (error) {
1126
        console.error('Error communicating with DeepSeek API: ', error);
1127
        if (!response.headersSent) {
1128
            response.send({ error: true });
1129
        } else {
1130
            response.end();
1131
        }
1132
    }
1133
}
1134

1135
/**
1136
 * Sends a request to XAI API.
1137
 * @param {express.Request} request Express request
1138
 * @param {express.Response} response Express response
1139
 */
1140
async function sendXaiRequest(request, response) {
1141
    const apiUrl = new URL(request.body.reverse_proxy || API_XAI).toString();
1142
    const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.XAI, request.body.secret_id);
1143

1144
    if (!apiKey && !request.body.reverse_proxy) {
1145
        console.warn('xAI API key is missing.');
1146
        return response.status(400).send({ error: true });
1147
    }
1148

1149
    const controller = new AbortController();
1150
    request.socket.removeAllListeners('close');
1151
    request.socket.on('close', function () {
1152
        controller.abort();
1153
    });
1154

1155
    try {
1156
        let bodyParams = {};
1157

1158
        if (request.body.logprobs > 0) {
1159
            bodyParams['top_logprobs'] = request.body.logprobs;
1160
            bodyParams['logprobs'] = true;
1161
        }
1162

1163
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
1164
            bodyParams['tools'] = request.body.tools;
1165
            bodyParams['tool_choice'] = request.body.tool_choice;
1166
        }
1167

1168
        if (Array.isArray(request.body.stop) && request.body.stop.length > 0) {
1169
            bodyParams['stop'] = request.body.stop;
1170
        }
1171

1172
        if (request.body.reasoning_effort) {
1173
            bodyParams['reasoning_effort'] = request.body.reasoning_effort === 'high' ? 'high' : 'low';
1174
        }
1175

1176
        if (request.body.json_schema) {
1177
            bodyParams['response_format'] = {
1178
                type: 'json_schema',
1179
                json_schema: {
1180
                    name: request.body.json_schema.name,
1181
                    strict: request.body.json_schema.strict ?? true,
1182
                    schema: request.body.json_schema.value,
1183
                },
1184
            };
1185
        }
1186

1187
        const processedMessages = request.body.messages = convertXAIMessages(request.body.messages, getPromptNames(request));
1188

1189
        const requestBody = {
1190
            'messages': processedMessages,
1191
            'model': request.body.model,
1192
            'temperature': request.body.temperature,
1193
            'max_tokens': request.body.max_tokens,
1194
            'max_completion_tokens': request.body.max_completion_tokens,
1195
            'stream': request.body.stream,
1196
            'presence_penalty': request.body.presence_penalty,
1197
            'frequency_penalty': request.body.frequency_penalty,
1198
            'top_p': request.body.top_p,
1199
            'seed': request.body.seed,
1200
            'n': request.body.n,
1201
            ...bodyParams,
1202
        };
1203

1204
        const config = {
1205
            method: 'POST',
1206
            headers: {
1207
                'Content-Type': 'application/json',
1208
                'Authorization': 'Bearer ' + apiKey,
1209
            },
1210
            body: JSON.stringify(requestBody),
1211
            signal: controller.signal,
1212
        };
1213

1214
        console.debug('xAI request:', requestBody);
1215

1216
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
1217

1218
        if (request.body.stream) {
1219
            await forwardFetchResponse(generateResponse, response);
1220
        } else {
1221
            if (!generateResponse.ok) {
1222
                const errorText = await generateResponse.text();
1223
                console.warn(`xAI API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
1224
                const errorJson = tryParse(errorText) ?? { error: true };
1225
                return response.status(500).send(errorJson);
1226
            }
1227
            const generateResponseJson = await generateResponse.json();
1228
            console.debug('xAI response:', generateResponseJson);
1229
            return response.send(generateResponseJson);
1230
        }
1231
    } catch (error) {
1232
        console.error('Error communicating with xAI API: ', error);
1233
        if (!response.headersSent) {
1234
            response.send({ error: true });
1235
        } else {
1236
            response.end();
1237
        }
1238
    }
1239
}
1240

1241
/**
1242
 * Sends a request to AI/ML API.
1243
 * @param {express.Request} request Express request
1244
 * @param {express.Response} response Express response
1245
 */
1246
async function sendAimlapiRequest(request, response) {
1247
    const apiUrl = API_AIMLAPI;
1248
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.AIMLAPI, request.body.secret_id);
1249

1250
    if (!apiKey) {
1251
        console.warn('AI/ML API key is missing.');
1252
        return response.status(400).send({ error: true });
1253
    }
1254

1255
    const controller = new AbortController();
1256
    request.socket.removeAllListeners('close');
1257
    request.socket.on('close', function () {
1258
        controller.abort();
1259
    });
1260

1261
    try {
1262
        let bodyParams = {};
1263

1264
        if (request.body.logprobs > 0) {
1265
            bodyParams['top_logprobs'] = request.body.logprobs;
1266
            bodyParams['logprobs'] = true;
1267
        }
1268

1269
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
1270
            bodyParams['tools'] = request.body.tools;
1271
            bodyParams['tool_choice'] = request.body.tool_choice;
1272
        }
1273

1274
        if (Array.isArray(request.body.stop) && request.body.stop.length > 0) {
1275
            bodyParams['stop'] = request.body.stop;
1276
        }
1277

1278
        if (request.body.reasoning_effort) {
1279
            bodyParams['reasoning_effort'] = request.body.reasoning_effort;
1280
        }
1281

1282
        if (request.body.json_schema) {
1283
            bodyParams['response_format'] = {
1284
                type: 'json_schema',
1285
                json_schema: {
1286
                    name: request.body.json_schema.name,
1287
                    description: request.body.json_schema.description,
1288
                    schema: request.body.json_schema.value,
1289
                    strict: request.body.json_schema.strict ?? true,
1290
                },
1291
            };
1292
        }
1293

1294
        const requestBody = {
1295
            'messages': request.body.messages,
1296
            'model': request.body.model,
1297
            'temperature': request.body.temperature,
1298
            'max_tokens': request.body.max_tokens,
1299
            'stream': request.body.stream,
1300
            'presence_penalty': request.body.presence_penalty,
1301
            'frequency_penalty': request.body.frequency_penalty,
1302
            'top_p': request.body.top_p,
1303
            'seed': request.body.seed,
1304
            'n': request.body.n,
1305
            ...bodyParams,
1306
        };
1307

1308
        const config = {
1309
            method: 'POST',
1310
            headers: {
1311
                'Content-Type': 'application/json',
1312
                'Authorization': 'Bearer ' + apiKey,
1313
                ...AIMLAPI_HEADERS,
1314
            },
1315
            body: JSON.stringify(requestBody),
1316
            signal: controller.signal,
1317
        };
1318

1319
        console.debug('AI/ML API request:', requestBody);
1320

1321
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
1322

1323
        if (request.body.stream) {
1324
            await forwardFetchResponse(generateResponse, response);
1325
        } else {
1326
            if (!generateResponse.ok) {
1327
                const errorText = await generateResponse.text();
1328
                console.warn(`AI/ML API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
1329
                const errorJson = tryParse(errorText) ?? { error: true };
1330
                return response.status(500).send(errorJson);
1331
            }
1332
            const generateResponseJson = await generateResponse.json();
1333
            console.debug('AI/ML API response:', generateResponseJson);
1334
            return response.send(generateResponseJson);
1335
        }
1336
    } catch (error) {
1337
        console.error('Error communicating with AI/ML API: ', error);
1338
        if (!response.headersSent) {
1339
            response.send({ error: true });
1340
        } else {
1341
            response.end();
1342
        }
1343
    }
1344
}
1345

1346
/**
1347
 * Sends a request to Electron Hub.
1348
 * @param {express.Request} request Express request
1349
 * @param {express.Response} response Express response
1350
 */
1351
async function sendElectronHubRequest(request, response) {
1352
    const apiUrl = API_ELECTRONHUB;
1353
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.ELECTRONHUB, request.body.secret_id);
1354

1355
    if (!apiKey) {
1356
        console.warn('Electron Hub key is missing.');
1357
        return response.status(400).send({ error: true });
1358
    }
1359

1360
    const controller = new AbortController();
1361
    request.socket.removeAllListeners('close');
1362
    request.socket.on('close', function () {
1363
        controller.abort();
1364
    });
1365

1366
    try {
1367
        let bodyParams = {};
1368

1369
        if (request.body.enable_web_search) {
1370
            bodyParams['web_search'] = true;
1371
        }
1372

1373
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
1374
            bodyParams['tools'] = request.body.tools;
1375
            bodyParams['tool_choice'] = request.body.tool_choice;
1376
        }
1377

1378
        if (request.body.reasoning_effort) {
1379
            bodyParams['reasoning_effort'] = request.body.reasoning_effort;
1380
        }
1381

1382
        if (request.body.json_schema) {
1383
            bodyParams['response_format'] = {
1384
                type: 'json_schema',
1385
                json_schema: {
1386
                    name: request.body.json_schema.name,
1387
                    description: request.body.json_schema.description,
1388
                    schema: request.body.json_schema.value,
1389
                    strict: request.body.json_schema.strict ?? true,
1390
                },
1391
            };
1392
        }
1393

1394
        const isClaude = /^claude-/.test(request.body.model);
1395

1396
        if (Array.isArray(request.body.messages) && isClaude) {
1397
            if (enableSystemPromptCache) {
1398
                cachingSystemPromptForOpenRouter(request.body.messages, cacheTTL);
1399
            }
1400

1401
            if (cachingAtDepth !== -1) {
1402
                cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
1403
            }
1404
        }
1405

1406
        const requestBody = {
1407
            'messages': request.body.messages,
1408
            'model': request.body.model,
1409
            'temperature': request.body.temperature,
1410
            'max_tokens': request.body.max_tokens,
1411
            'stream': request.body.stream,
1412
            'presence_penalty': request.body.presence_penalty,
1413
            'frequency_penalty': request.body.frequency_penalty,
1414
            'top_p': request.body.top_p,
1415
            'top_k': request.body.top_k,
1416
            'logit_bias': request.body.logit_bias,
1417
            'seed': request.body.seed,
1418
            ...bodyParams,
1419
        };
1420

1421
        const config = {
1422
            method: 'POST',
1423
            headers: {
1424
                'Content-Type': 'application/json',
1425
                'Authorization': 'Bearer ' + apiKey,
1426
            },
1427
            body: JSON.stringify(requestBody),
1428
            signal: controller.signal,
1429
        };
1430

1431
        console.debug('Electron Hub request:', requestBody);
1432

1433
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
1434

1435
        if (request.body.stream) {
1436
            await forwardFetchResponse(generateResponse, response);
1437
        } else {
1438
            if (!generateResponse.ok) {
1439
                const errorText = await generateResponse.text();
1440
                console.warn('Electron Hub returned error: ', errorText);
1441
                const errorJson = tryParse(errorText) ?? { error: true };
1442
                return response.status(500).send(errorJson);
1443
            }
1444
            const generateResponseJson = await generateResponse.json();
1445
            console.debug('Electron Hub response:', generateResponseJson);
1446
            return response.send(generateResponseJson);
1447
        }
1448
    } catch (error) {
1449
        console.error('Error communicating with Electron Hub: ', error);
1450
        if (!response.headersSent) {
1451
            response.send({ error: true });
1452
        } else {
1453
            response.end();
1454
        }
1455
    }
1456
}
1457

1458
/**
1459
 * Sends a request to Chutes.
1460
 * @param {express.Request} request Express request
1461
 * @param {express.Response} response Express response
1462
 */
1463
async function sendChutesRequest(request, response) {
1464
    const apiUrl = API_CHUTES;
1465
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.CHUTES, request.body.secret_id);
1466

1467
    if (!apiKey) {
1468
        console.warn('Chutes key is missing.');
1469
        return response.status(400).send({ error: true });
1470
    }
1471

1472
    const controller = new AbortController();
1473
    request.socket.removeAllListeners('close');
1474
    request.socket.on('close', function () {
1475
        controller.abort();
1476
    });
1477

1478
    try {
1479
        let bodyParams = {};
1480

1481
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
1482
            bodyParams['tools'] = request.body.tools;
1483
            bodyParams['tool_choice'] = request.body.tool_choice;
1484
        }
1485

1486
        if (request.body.logprobs > 0) {
1487
            bodyParams['top_logprobs'] = request.body.logprobs;
1488
            bodyParams['logprobs'] = true;
1489
        }
1490

1491
        if (request.body.json_schema) {
1492
            bodyParams['response_format'] = {
1493
                type: 'json_schema',
1494
                json_schema: {
1495
                    name: request.body.json_schema.name,
1496
                    description: request.body.json_schema.description,
1497
                    schema: request.body.json_schema.value,
1498
                    strict: request.body.json_schema.strict ?? true,
1499
                },
1500
            };
1501
        }
1502

1503
        const requestBody = {
1504
            'messages': request.body.messages,
1505
            'model': request.body.model,
1506
            'temperature': request.body.temperature,
1507
            'max_tokens': request.body.max_tokens,
1508
            'stream': request.body.stream,
1509
            'presence_penalty': request.body.presence_penalty,
1510
            'frequency_penalty': request.body.frequency_penalty,
1511
            'repetition_penalty': request.body.repetition_penalty,
1512
            'min_p': request.body.min_p,
1513
            'top_p': request.body.top_p,
1514
            'top_k': request.body.top_k,
1515
            'seed': request.body.seed,
1516
            'stop': request.body.stop,
1517
            'reasoning_effort': request.body.reasoning_effort,
1518
            'logit_bias': request.body.logit_bias,
1519
            ...bodyParams,
1520
        };
1521

1522
        const config = {
1523
            method: 'POST',
1524
            headers: {
1525
                'Content-Type': 'application/json',
1526
                'Authorization': 'Bearer ' + apiKey,
1527
            },
1528
            body: JSON.stringify(requestBody),
1529
            signal: controller.signal,
1530
        };
1531

1532
        console.debug('Chutes request:', requestBody);
1533

1534
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
1535

1536
        if (request.body.stream) {
1537
            await forwardFetchResponse(generateResponse, response);
1538
        } else {
1539
            if (!generateResponse.ok) {
1540
                const errorText = await generateResponse.text();
1541
                console.warn('Chutes returned error: ', errorText);
1542
                const errorJson = tryParse(errorText) ?? { error: true };
1543
                return response.status(500).send(errorJson);
1544
            }
1545
            const generateResponseJson = await generateResponse.json();
1546
            console.debug('Chutes response:', generateResponseJson);
1547
            return response.send(generateResponseJson);
1548
        }
1549
    } catch (error) {
1550
        console.error('Error communicating with Chutes: ', error);
1551
        if (!response.headersSent) {
1552
            response.send({ error: true });
1553
        } else {
1554
            response.end();
1555
        }
1556
    }
1557
}
1558

1559
/**
1560
 * Sends a request to MiniMax.
1561
 * @param {express.Request} request Express request
1562
 * @param {express.Response} response Express response
1563
 */
1564
async function sendMinimaxRequest(request, response) {
1565
    const apiUrl = request.body.minimax_endpoint === MINIMAX_ENDPOINT.CN
1566
        ? API_MINIMAX_CN : API_MINIMAX;
1567
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.MINIMAX, request.body.secret_id);
1568

1569
    if (!apiKey) {
1570
        console.warn('MiniMax key is missing.');
1571
        return response.status(400).send({ error: true });
1572
    }
1573

1574
    const controller = new AbortController();
1575
    request.socket.removeAllListeners('close');
1576
    request.socket.on('close', function () {
1577
        controller.abort();
1578
    });
1579

1580
    try {
1581
        // MiniMax does not allow consecutive messages with the same role.
1582
        // Merge them into a single message to avoid "invalid chat setting (2013)".
1583
        const messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.MERGE_TOOLS, getPromptNames(request));
1584

1585
        let bodyParams = {};
1586

1587
        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
1588
            bodyParams['tools'] = request.body.tools;
1589
            bodyParams['tool_choice'] = request.body.tool_choice;
1590
        }
1591

1592
        const requestBody = {
1593
            'messages': messages,
1594
            'model': request.body.model,
1595
            'temperature': request.body.temperature,
1596
            'max_tokens': request.body.model === 'M2-her' ? Math.min(request.body.max_tokens, 2048) : request.body.max_tokens,
1597
            'stream': request.body.stream,
1598
            'top_p': request.body.top_p,
1599
            'stop': request.body.stop,
1600
            ...bodyParams,
1601
        };
1602

1603
        const config = {
1604
            method: 'POST',
1605
            headers: {
1606
                'Content-Type': 'application/json',
1607
                'Authorization': 'Bearer ' + apiKey,
1608
            },
1609
            body: JSON.stringify(requestBody),
1610
            signal: controller.signal,
1611
        };
1612

1613
        console.debug('MiniMax request:', requestBody);
1614

1615
        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
1616

1617
        if (request.body.stream) {
1618
            await forwardFetchResponse(generateResponse, response);
1619
        } else {
1620
            if (!generateResponse.ok) {
1621
                const errorText = await generateResponse.text();
1622
                console.warn('MiniMax returned error: ', errorText);
1623
                const errorJson = tryParse(errorText) ?? { error: true };
1624
                return response.status(500).send(errorJson);
1625
            }
1626
            const generateResponseJson = await generateResponse.json();
1627
            console.debug('MiniMax response:', generateResponseJson);
1628
            return response.send(generateResponseJson);
1629
        }
1630
    } catch (error) {
1631
        console.error('Error communicating with MiniMax: ', error);
1632
        if (!response.headersSent) {
1633
            response.send({ error: true });
1634
        } else {
1635
            response.end();
1636
        }
1637
    }
1638
}
1639

1640
/**
1641
 * @param {express.Request} request Express request object (contains request.body with all generate_data)
1642
 * @param {express.Response} response Express response object
1643
 */
1644
async function sendAzureOpenAIRequest(request, response) {
1645
    // 1. GATHER & VALIDATE SETTINGS
1646
    const { azure_base_url, azure_deployment_name, azure_api_version } = request.body;
1647
    const apiKey = readSecret(request.user.directories, SECRET_KEYS.AZURE_OPENAI, request.body.secret_id);
1648
    if (!azure_base_url || !azure_deployment_name || !azure_api_version || !apiKey) {
1649
        return response.status(400).send({
1650
            error: {
1651
                message: 'Azure OpenAI configuration is incomplete. Please provide Base URL, Deployment Name, API Version, and API Key in the connection settings.',
1652
            },
1653
        });
1654
    }
1655

1656
    // 2. PREPARE THE REQUEST
1657
    const url = new URL(`/openai/deployments/${azure_deployment_name}/chat/completions`, azure_base_url);
1658
    url.searchParams.set('api-version', azure_api_version);
1659
    const endpointUrl = url.toString();
1660

1661
    // Create the base payload with all standard parameters
1662
    const apiRequestBody = /** @type {any} */ ({});
1663
    for (const key of AZURE_OPENAI_KEYS) {
1664
        if (Object.hasOwn(request.body, key)) {
1665
            apiRequestBody[key] = request.body[key];
1666
        }
1667
    }
1668

1669
    // Handle Structured Output (JSON Mode) by translating the custom `json_schema` object.
1670
    if (request.body.json_schema) {
1671
        apiRequestBody['response_format'] = {
1672
            type: 'json_schema',
1673
            json_schema: {
1674
                name: request.body.json_schema.name,
1675
                strict: request.body.json_schema.strict ?? true,
1676
                schema: request.body.json_schema.value,
1677
            },
1678
        };
1679
    }
1680

1681
    // Adjust logprobs for Azure OpenAI, which follows the OpenAI Chat Completions API spec.
1682
    if (typeof apiRequestBody.logprobs === 'number' && apiRequestBody.logprobs > 0) {
1683
        apiRequestBody.top_logprobs = apiRequestBody.logprobs;
1684
        apiRequestBody.logprobs = true;
1685
    }
1686

1687
    // Do not send reasoning effort to models which do not support it
1688
    apiRequestBody['reasoning_effort'] = OPENAI_REASONING_EFFORT_MODELS.includes(request.body.model)
1689
        ? OPENAI_FIXED_REASONING_EFFORT[request.body.model] ?? OPENAI_REASONING_EFFORT_MAP[request.body.reasoning_effort] ?? request.body.reasoning_effort
1690
        : undefined;
1691

1692
    const controller = new AbortController();
1693
    request.socket.removeAllListeners('close');
1694
    request.socket.on('close', () => controller.abort());
1695

1696
    const config = {
1697
        method: 'POST',
1698
        headers: {
1699
            'Content-Type': 'application/json',
1700
            'api-key': apiKey,
1701
        },
1702
        body: JSON.stringify(apiRequestBody),
1703
        signal: controller.signal,
1704
    };
1705

1706
    console.info(`Sending request to Azure OpenAI: ${endpointUrl}`);
1707
    console.debug('Azure OpenAI Request Body:', apiRequestBody);
1708
    try {
1709
        const fetchResponse = await fetch(endpointUrl, config);
1710

1711
        if (request.body.stream) {
1712
            return await forwardFetchResponse(fetchResponse, response);
1713
        }
1714

1715
        if (fetchResponse.ok) {
1716
            /** @type {any} */
1717
            const json = await fetchResponse.json();
1718
            console.debug('Azure OpenAI response:', json);
1719
            return response.send(json);
1720
        }
1721

1722
        const text = await fetchResponse.text();
1723
        const data = tryParse(text) || { error: { message: fetchResponse.statusText || 'Unknown error occurred' } };
1724
        return response.status(500).send(data);
1725
    } catch (error) {
1726
        const message = error.name === 'AbortError'
1727
            ? 'Request was aborted by the client.'
1728
            : (error.message || 'An unknown network error occurred.');
1729
        return response.status(500).send({ error: { message, ...error } });
1730
    }
1731
}
1732

1733
export const router = express.Router();
1734

1735
router.post('/status', async function (request, statusResponse) {
1736
    try {
1737
        if (!request.body) return statusResponse.sendStatus(400);
1738

1739
        let apiUrl = '';
1740
        let apiKey = '';
1741
        let headers = {};
1742
        let queryParams = {};
1743

1744
        if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1745
            apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1746
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI, request.body.secret_id);
1747
            headers = {};
1748
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1749
            apiUrl = 'https://openrouter.ai/api/v1';
1750
            apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER, request.body.secret_id);
1751
            // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1752
            headers = { ...OPENROUTER_HEADERS };
1753
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
1754
            apiUrl = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
1755
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI, request.body.secret_id);
1756
            headers = {};
1757
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
1758
            apiUrl = request.body.custom_url;
1759
            apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM, request.body.secret_id);
1760
            headers = {};
1761
            mergeObjectWithYaml(headers, request.body.custom_include_headers);
1762
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {
1763
            apiUrl = API_COHERE_V1;
1764
            apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE, request.body.secret_id);
1765
            headers = {};
1766
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CHUTES) {
1767
            apiUrl = API_CHUTES;
1768
            apiKey = readSecret(request.user.directories, SECRET_KEYS.CHUTES, request.body.secret_id);
1769
            headers = {};
1770
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ELECTRONHUB) {
1771
            apiUrl = API_ELECTRONHUB;
1772
            apiKey = readSecret(request.user.directories, SECRET_KEYS.ELECTRONHUB, request.body.secret_id);
1773
            headers = {};
1774
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
1775
            apiUrl = API_NANOGPT;
1776
            apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT, request.body.secret_id);
1777
            headers = {};
1778
            queryParams = { detailed: true };
1779
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {
1780
            apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK.replace('/beta', '')).toString();
1781
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK, request.body.secret_id);
1782
            headers = {};
1783
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.XAI) {
1784
            apiUrl = new URL(request.body.reverse_proxy || API_XAI).toString();
1785
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.XAI, request.body.secret_id);
1786
            headers = {};
1787
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AIMLAPI) {
1788
            apiUrl = API_AIMLAPI;
1789
            apiKey = readSecret(request.user.directories, SECRET_KEYS.AIMLAPI, request.body.secret_id);
1790
            headers = { ...AIMLAPI_HEADERS };
1791
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
1792
            apiUrl = 'https://gen.pollinations.ai/text';
1793
            apiKey = readSecret(request.user.directories, SECRET_KEYS.POLLINATIONS, request.body.secret_id);
1794
            headers = {};
1795
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
1796
            apiUrl = API_GROQ;
1797
            apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ, request.body.secret_id);
1798
            headers = {};
1799
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
1800
            apiUrl = API_COMETAPI;
1801
            apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI, request.body.secret_id);
1802
            headers = {};
1803
            throw new Error('This provider is temporarily disabled.');
1804
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
1805
            apiUrl = new URL(request.body.reverse_proxy || API_MOONSHOT).toString();
1806
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MOONSHOT, request.body.secret_id);
1807
            headers = {};
1808
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
1809
            apiUrl = API_FIREWORKS;
1810
            apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS, request.body.secret_id);
1811
            headers = {};
1812
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MAKERSUITE) {
1813
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE, request.body.secret_id);
1814
            apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
1815
            const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
1816
            const modelsUrl = !apiKey && request.body.reverse_proxy
1817
                ? `${apiUrl}/${apiVersion}/models`
1818
                : `${apiUrl}/${apiVersion}/models?key=${apiKey}`;
1819

1820
            if (!apiKey && !request.body.reverse_proxy) {
1821
                console.warn('Google AI Studio API key is missing.');
1822
                return statusResponse.status(400).send({ error: true });
1823
            }
1824

1825
            try {
1826
                const response = await fetch(modelsUrl);
1827

1828
                if (response.ok) {
1829
                    /** @type {any} */
1830
                    const data = await response.json();
1831
                    // Transform Google AI Studio models to OpenAI format
1832
                    const models = data.models
1833
                        ?.filter(model => model.supportedGenerationMethods?.includes('generateContent'))
1834
                        ?.map(model => ({
1835
                            ...model,
1836
                            id: model.name.replace('models/', ''),
1837
                        })) || [];
1838

1839
                    console.info('Available Google AI Studio models:', models.map(m => m.id));
1840
                    return statusResponse.send({ data: models });
1841
                } else {
1842
                    console.warn('Google AI Studio models endpoint failed:', response.status, response.statusText);
1843
                    return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
1844
                }
1845
            } catch (error) {
1846
                console.error('Error fetching Google AI Studio models:', error);
1847
                return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
1848
            }
1849
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AZURE_OPENAI) {
1850
            const { azure_base_url, azure_deployment_name, azure_api_version } = request.body;
1851
            const apiKey = readSecret(request.user.directories, SECRET_KEYS.AZURE_OPENAI, request.body.secret_id);
1852

1853
            // 1) Validate configuration from the frontend
1854
            if (!apiKey || !azure_base_url || !azure_deployment_name || !azure_api_version) {
1855
                console.warn('Azure OpenAI status check failed: missing config from frontend.');
1856
                return statusResponse.status(400).send({ error: true, message: 'Azure configuration is incomplete.' });
1857
            }
1858
            // 2) Build URLs using the URL API for consistency and robustness.
1859
            const modelsUrl = new URL('/openai/models', azure_base_url);
1860
            modelsUrl.searchParams.set('api-version', azure_api_version);
1861

1862
            const chatUrl = new URL(`/openai/deployments/${azure_deployment_name}/chat/completions`, azure_base_url);
1863
            chatUrl.searchParams.set('api-version', azure_api_version);
1864

1865
            // Map common status codes to user-friendly error messages
1866
            const azureStatusErrorMap = {
1867
                400: 'API version may be invalid for this resource.',
1868
                401: 'Invalid API key or insufficient permissions.',
1869
                403: 'Invalid API key or insufficient permissions.',
1870
                404: 'Endpoint URL appears incorrect (404).',
1871
            };
1872

1873
            try {
1874
                // ---- A) GET /models: fast sanity check for endpoint + api key + api version ----
1875
                const apiConfigTest = await fetch(modelsUrl, {
1876
                    method: 'GET',
1877
                    headers: { 'api-key': apiKey, 'Accept': 'application/json' },
1878
                });
1879

1880
                if (!apiConfigTest.ok) {
1881
                    let errText = '';
1882
                    try { errText = await apiConfigTest.text(); } catch { /* response body may be empty */ }
1883

1884
                    console.warn('Azure OpenAI GET /models failed:', apiConfigTest.status, apiConfigTest.statusText, errText || '');
1885

1886
                    const defaultMessage = `Azure Models endpoint error: ${apiConfigTest.statusText}`;
1887
                    const message = azureStatusErrorMap[apiConfigTest.status] ?? defaultMessage;
1888
                    return statusResponse.status(apiConfigTest.status).send({ error: true, message });
1889
                }
1890

1891
                // ---- B) POST /chat/completions: verify deployment + read underlying model ID ----
1892
                // Small, deterministic probe to minimize cost/latency
1893
                const modelPayload = {
1894
                    messages: [{ role: 'user', content: 'Say word Hi' }],
1895
                    stream: false,
1896
                    max_completion_tokens: 5,
1897
                };
1898

1899
                const modelRequest = await fetch(chatUrl, {
1900
                    method: 'POST',
1901
                    headers: { 'api-key': apiKey, 'Content-Type': 'application/json', 'Accept': 'application/json' },
1902
                    body: JSON.stringify(modelPayload),
1903
                });
1904

1905
                let modelResponse;
1906
                try {
1907
                    modelResponse = await modelRequest.json();
1908
                } catch {
1909
                    modelResponse = { raw: 'Failed to parse JSON response from chat completions probe.' };
1910
                }
1911

1912
                const modelId = /** @type {any} */ (modelResponse)?.model;
1913
                if (!modelId) {
1914
                    console.warn('Azure status check succeeded but could not find a model ID in the response.');
1915
                    console.debug('Azure Response Body:', modelResponse);
1916
                    // Keep a benign success to avoid UX disruption in the UI
1917
                    return statusResponse.send({ data: [] });
1918
                }
1919

1920
                console.info(color.green('Azure OpenAI connection successful. Detected model:'), modelId);
1921
                // Consistent response format: always an array of { id }
1922
                return statusResponse.send({ data: [{ id: modelId }] });
1923
            } catch (error) {
1924
                console.error('Azure OpenAI status check connection error:', error);
1925
                return statusResponse.status(500).send({ error: true, message: 'Failed to connect to the Azure endpoint.' });
1926
            }
1927
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
1928
            const defaultApiUrl = request.body.siliconflow_endpoint === SILICONFLOW_ENDPOINT.CN
1929
                ? API_SILICONFLOW_CN : API_SILICONFLOW;
1930
            apiUrl = defaultApiUrl;
1931
            apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW, request.body.secret_id);
1932
            headers = {};
1933
            queryParams = { type: 'text', sub_type: 'chat' };
1934
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.WORKERS_AI) {
1935
            apiKey = readSecret(request.user.directories, SECRET_KEYS.WORKERS_AI, request.body.secret_id);
1936

1937
            if (!apiKey) {
1938
                console.warn('Cloudflare Workers AI API key is missing.');
1939
                return statusResponse.status(400).send({ error: true });
1940
            }
1941

1942
            try {
1943
                const accountId = String(request.body.workers_ai_account_id || '').trim();
1944
                if (!accountId) {
1945
                    console.warn('Cloudflare Workers AI Account ID is missing.');
1946
                    return statusResponse.status(400).send({ error: true });
1947
                }
1948

1949
                const modelsUrl = new URL(`${API_WORKERS_AI}/${encodeURIComponent(accountId)}/ai/models/search`);
1950
                modelsUrl.searchParams.set('task', 'Text Generation');
1951
                modelsUrl.searchParams.set('per_page', '1000');
1952

1953
                const response = await fetch(modelsUrl, {
1954
                    method: 'GET',
1955
                    headers: {
1956
                        'Authorization': 'Bearer ' + apiKey,
1957
                    },
1958
                });
1959

1960
                if (response.ok) {
1961
                    /** @type {any} */
1962
                    const data = await response.json();
1963
                    const models = Array.isArray(data?.result)
1964
                        ? data.result.map(model => ({ ...model, id: model.name }))
1965
                        : [];
1966

1967
                    console.debug('Available Cloudflare Workers AI models:', models.map(m => m.id));
1968
                    return statusResponse.send({ data: models });
1969
                } else {
1970
                    console.warn('Cloudflare Workers AI models endpoint failed:', response.status, response.statusText);
1971
                    return statusResponse.status(response.status).send({ error: true });
1972
                }
1973
            } catch (error) {
1974
                console.error('Error fetching Cloudflare Workers AI models:', error);
1975
                return statusResponse.status(500).send({ error: true });
1976
            }
1977
        } else {
1978
            console.warn('This chat completion source is not supported yet.');
1979
            return statusResponse.status(400).send({ error: true });
1980
        }
1981

1982
        if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
1983
            console.warn('Chat Completion API key is missing.');
1984
            return statusResponse.status(400).send({ error: true });
1985
        }
1986

1987
        const modelsUrl = new URL(urlJoin(apiUrl, '/models'));
1988
        Object.keys(queryParams).forEach(key => {
1989
            modelsUrl.searchParams.append(key, queryParams[key]);
1990
        });
1991
        const response = await fetch(modelsUrl, {
1992
            method: 'GET',
1993
            headers: {
1994
                'Authorization': 'Bearer ' + apiKey,
1995
                ...headers,
1996
            },
1997
        });
1998

1999
        if (response.ok) {
2000
            /** @type {any} */
2001
            let data = await response.json();
2002

2003
            if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS && Array.isArray(data)) {
2004
                data = { data: data.map(model => ({ id: model.name, ...model })) };
2005
            }
2006

2007
            if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CHUTES && Array.isArray(data?.data)) {
2008
                data.data = data.data
2009
                    .filter(model => model?.id)
2010
                    .map(model => {
2011
                        if (model.pricing?.prompt !== undefined && model.pricing?.completion !== undefined) {
2012
                            return {
2013
                                ...model,
2014
                                pricing: {
2015
                                    ...model.pricing,
2016
                                    input: model.pricing.prompt,
2017
                                    output: model.pricing.completion,
2018
                                },
2019
                            };
2020
                        }
2021
                        return model;
2022
                    });
2023
            }
2024

2025
            statusResponse.send(data);
2026

2027
            if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE && Array.isArray(data?.models)) {
2028
                data.data = data.models.map(model => ({ id: model.name, ...model }));
2029
            }
2030

2031
            if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER && Array.isArray(data?.data)) {
2032
                let models = [];
2033

2034
                data.data.forEach(model => {
2035
                    const context_length = model.context_length;
2036
                    const tokens_dollar = Number(1 / (1000 * model.pricing?.prompt));
2037
                    const tokens_rounded = (Math.round(tokens_dollar * 1000) / 1000).toFixed(0);
2038
                    models[model.id] = {
2039
                        tokens_per_dollar: tokens_rounded + 'k',
2040
                        context_length: context_length,
2041
                    };
2042
                });
2043

2044
                console.info('Available OpenRouter models:', models);
2045
            } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
2046
                const models = data?.data;
2047
                console.info(models);
2048
            } else {
2049
                const models = data?.data;
2050

2051
                if (Array.isArray(models)) {
2052
                    const modelIds = models.filter(x => x && typeof x === 'object').map(x => x.id).sort();
2053
                    console.info('Available models:', modelIds);
2054
                } else {
2055
                    console.warn('Chat Completion endpoint did not return a list of models.');
2056
                }
2057
            }
2058
        } else {
2059
            console.error('Chat Completion status check failed. Either Access Token is incorrect or API endpoint is down.');
2060
            statusResponse.send({ error: true, data: { data: [] } });
2061
        }
2062
    } catch (e) {
2063
        console.error(e);
2064

2065
        if (!statusResponse.headersSent) {
2066
            statusResponse.send({ error: true });
2067
        } else {
2068
            statusResponse.end();
2069
        }
2070
    }
2071
});
2072

2073
router.post('/bias', async function (request, response) {
2074
    if (!request.body || !Array.isArray(request.body))
2075
        return response.sendStatus(400);
2076

2077
    try {
2078
        const result = {};
2079
        const model = getTokenizerModel(String(request.query.model || ''));
2080

2081
        // no bias for claude
2082
        if (model == 'claude') {
2083
            return response.send(result);
2084
        }
2085

2086
        let encodeFunction;
2087

2088
        if (sentencepieceTokenizers.includes(model)) {
2089
            const tokenizer = getSentencepiceTokenizer(model);
2090
            const instance = await tokenizer?.get();
2091
            if (!instance) {
2092
                console.error('Tokenizer not initialized:', model);
2093
                return response.send({});
2094
            }
2095
            encodeFunction = (text) => new Uint32Array(instance.encodeIds(text));
2096
        } else if (webTokenizers.includes(model)) {
2097
            const tokenizer = getWebTokenizer(model);
2098
            const instance = await tokenizer?.get();
2099
            if (!instance) {
2100
                console.warn('Tokenizer not initialized:', model);
2101
                return response.send({});
2102
            }
2103
            encodeFunction = (text) => new Uint32Array(instance.encode(text));
2104
        } else {
2105
            const tokenizer = getTiktokenTokenizer(model);
2106
            encodeFunction = (tokenizer.encode.bind(tokenizer));
2107
        }
2108

2109
        for (const entry of request.body) {
2110
            if (!entry || !entry.text) {
2111
                continue;
2112
            }
2113

2114
            try {
2115
                const tokens = getEntryTokens(entry.text, encodeFunction);
2116

2117
                for (const token of tokens) {
2118
                    result[token] = entry.value;
2119
                }
2120
            } catch {
2121
                console.warn('Tokenizer failed to encode:', entry.text);
2122
            }
2123
        }
2124

2125
        // not needed for cached tokenizers
2126
        //tokenizer.free();
2127
        return response.send(result);
2128

2129
        /**
2130
         * Gets tokenids for a given entry
2131
         * @param {string} text Entry text
2132
         * @param {(string) => Uint32Array} encode Function to encode text to token ids
2133
         * @returns {Uint32Array} Array of token ids
2134
         */
2135
        function getEntryTokens(text, encode) {
2136
            // Get raw token ids from JSON array
2137
            if (text.trim().startsWith('[') && text.trim().endsWith(']')) {
2138
                try {
2139
                    const json = JSON.parse(text);
2140
                    if (Array.isArray(json) && json.every(x => typeof x === 'number')) {
2141
                        return new Uint32Array(json);
2142
                    }
2143
                } catch {
2144
                    // ignore
2145
                }
2146
            }
2147

2148
            // Otherwise, get token ids from tokenizer
2149
            return encode(text);
2150
        }
2151
    } catch (error) {
2152
        console.error(error);
2153
        return response.send({});
2154
    }
2155
});
2156

2157
router.post('/generate', async function (request, response) {
2158
    try {
2159
        if (!request.body) return response.status(400).send({ error: true });
2160

2161
        const postProcessingType = request.body.custom_prompt_post_processing;
2162
        if (Array.isArray(request.body.messages) && postProcessingType) {
2163
            console.info('Applying custom prompt post-processing of type', postProcessingType);
2164
            request.body.messages = postProcessPrompt(
2165
                request.body.messages,
2166
                postProcessingType,
2167
                getPromptNames(request));
2168
        }
2169

2170
        if (request.body.json_schema?.value) {
2171
            request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
2172
        }
2173

2174
        switch (request.body.chat_completion_source) {
2175
            case CHAT_COMPLETION_SOURCES.CLAUDE: return await sendClaudeRequest(request, response);
2176
            case CHAT_COMPLETION_SOURCES.AI21: return await sendAI21Request(request, response);
2177
            case CHAT_COMPLETION_SOURCES.MAKERSUITE: return await sendMakerSuiteRequest(request, response);
2178
            case CHAT_COMPLETION_SOURCES.VERTEXAI: return await sendMakerSuiteRequest(request, response);
2179
            case CHAT_COMPLETION_SOURCES.MISTRALAI: return await sendMistralAIRequest(request, response);
2180
            case CHAT_COMPLETION_SOURCES.COHERE: return await sendCohereRequest(request, response);
2181
            case CHAT_COMPLETION_SOURCES.DEEPSEEK: return await sendDeepSeekRequest(request, response);
2182
            case CHAT_COMPLETION_SOURCES.AIMLAPI: return await sendAimlapiRequest(request, response);
2183
            case CHAT_COMPLETION_SOURCES.XAI: return await sendXaiRequest(request, response);
2184
            case CHAT_COMPLETION_SOURCES.CHUTES: return await sendChutesRequest(request, response);
2185
            case CHAT_COMPLETION_SOURCES.MINIMAX: return await sendMinimaxRequest(request, response);
2186
            case CHAT_COMPLETION_SOURCES.ELECTRONHUB: return await sendElectronHubRequest(request, response);
2187
            case CHAT_COMPLETION_SOURCES.AZURE_OPENAI: return await sendAzureOpenAIRequest(request, response);
2188
        }
2189

2190
        let apiUrl;
2191
        let apiKey;
2192
        let headers;
2193
        let bodyParams;
2194
        const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model)) || typeof request.body.messages === 'string';
2195

2196
        if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
2197
            apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
2198
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI, request.body.secret_id);
2199
            headers = {};
2200
            bodyParams = {
2201
                logprobs: request.body.logprobs,
2202
                top_logprobs: undefined,
2203
            };
2204

2205
            // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
2206
            if (!isTextCompletion && bodyParams.logprobs > 0) {
2207
                bodyParams.top_logprobs = bodyParams.logprobs;
2208
                bodyParams.logprobs = true;
2209
            }
2210

2211
            if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {
2212
                bodyParams['user'] = uuidv4();
2213
            }
2214

2215
            embedOpenRouterMedia(request.body.messages, { audio: true, video: false });
2216
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
2217
            apiUrl = 'https://openrouter.ai/api/v1';
2218
            apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER, request.body.secret_id);
2219
            // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
2220
            headers = { ...OPENROUTER_HEADERS };
2221
            const includeReasoning = Boolean(request.body.include_reasoning);
2222
            bodyParams = {
2223
                transforms: getOpenRouterTransforms(request),
2224
                plugins: getOpenRouterPlugins(request),
2225
                reasoning: {
2226
                    exclude: !includeReasoning,
2227
                },
2228
            };
2229

2230
            if (request.body.min_p !== undefined) {
2231
                bodyParams['min_p'] = request.body.min_p;
2232
            }
2233

2234
            if (request.body.top_a !== undefined) {
2235
                bodyParams['top_a'] = request.body.top_a;
2236
            }
2237

2238
            if (request.body.repetition_penalty !== undefined) {
2239
                bodyParams['repetition_penalty'] = request.body.repetition_penalty;
2240
            }
2241

2242
            if (Array.isArray(request.body.provider) && request.body.provider.length > 0) {
2243
                bodyParams['provider'] = {
2244
                    allow_fallbacks: request.body.allow_fallbacks ?? true,
2245
                    order: request.body.provider ?? [],
2246
                };
2247
            }
2248

2249
            if (Array.isArray(request.body.quantizations) && request.body.quantizations.length > 0) {
2250
                bodyParams['provider'] ??= {};
2251
                bodyParams['provider']['quantizations'] = request.body.quantizations;
2252
            }
2253

2254
            if (request.body.use_fallback) {
2255
                bodyParams['route'] = 'fallback';
2256
            }
2257

2258
            if (request.body.reasoning_effort) {
2259
                bodyParams['reasoning']['effort'] = request.body.reasoning_effort;
2260
            }
2261

2262
            if (request.body.verbosity) {
2263
                bodyParams['verbosity'] = request.body.verbosity;
2264
            }
2265

2266
            if (request.body.json_schema) {
2267
                bodyParams['response_format'] = {
2268
                    type: 'json_schema',
2269
                    json_schema: {
2270
                        name: request.body.json_schema.name,
2271
                        strict: request.body.json_schema.strict ?? true,
2272
                        schema: request.body.json_schema.value,
2273
                    },
2274
                };
2275
            }
2276

2277
            const isClaude = /^anthropic\/claude/.test(request.body.model);
2278
            const isGemini = /google\/gemini/.test(request.body.model);
2279
            const isCacheableGemini = isGemini && await isOpenRouterModelCacheable(request.body.model);
2280
            const enableGeminiSystemPromptCache = getConfigValue('gemini.enableSystemPromptCache', false, 'boolean');
2281

2282
            if (Array.isArray(request.body.messages)) {
2283
                embedOpenRouterMedia(request.body.messages, { audio: true, video: true });
2284
                addOpenRouterSignatures(request.body.messages, request.body.model);
2285

2286
                if (isClaude) {
2287
                    if (enableSystemPromptCache) {
2288
                        cachingSystemPromptForOpenRouter(request.body.messages, cacheTTL);
2289
                    }
2290

2291
                    if (cachingAtDepth !== -1) {
2292
                        cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
2293
                    }
2294
                }
2295

2296
                if (isCacheableGemini && enableGeminiSystemPromptCache) {
2297
                    cachingSystemPromptForOpenRouter(request.body.messages);
2298
                }
2299
            }
2300

2301
            if (isGemini) {
2302
                bodyParams['safety_settings'] = GEMINI_SAFETY;
2303
            }
2304
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2305
            apiUrl = request.body.custom_url;
2306
            apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM, request.body.secret_id);
2307
            headers = {};
2308
            bodyParams = {
2309
                logprobs: request.body.logprobs,
2310
                top_logprobs: undefined,
2311
            };
2312

2313
            // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
2314
            if (!isTextCompletion && bodyParams.logprobs > 0) {
2315
                bodyParams.top_logprobs = bodyParams.logprobs;
2316
                bodyParams.logprobs = true;
2317
            }
2318

2319
            mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
2320
            mergeObjectWithYaml(headers, request.body.custom_include_headers);
2321
            embedOpenRouterMedia(request.body.messages, { audio: true, video: false });
2322
            if (request.body.json_schema) {
2323
                bodyParams['response_format'] = {
2324
                    type: 'json_schema',
2325
                    json_schema: {
2326
                        name: request.body.json_schema.name,
2327
                        strict: request.body.json_schema.strict ?? true,
2328
                        schema: request.body.json_schema.value,
2329
                    },
2330
                };
2331
            }
2332
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
2333
            apiUrl = API_PERPLEXITY;
2334
            apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY, request.body.secret_id);
2335
            headers = {};
2336
            bodyParams = {
2337
                reasoning_effort: request.body.reasoning_effort,
2338
            };
2339
            request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
2340
            if (request.body.json_schema) {
2341
                bodyParams['response_format'] = {
2342
                    type: 'json_schema',
2343
                    json_schema: {
2344
                        schema: request.body.json_schema.value,
2345
                    },
2346
                };
2347
            }
2348
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
2349
            apiUrl = API_GROQ;
2350
            apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ, request.body.secret_id);
2351
            headers = {};
2352
            bodyParams = {};
2353
            if (request.body.json_schema) {
2354
                bodyParams['response_format'] = {
2355
                    type: 'json_schema',
2356
                    json_schema: {
2357
                        name: request.body.json_schema.name,
2358
                        description: request.body.json_schema.description,
2359
                        schema: request.body.json_schema.value,
2360
                        strict: request.body.json_schema.strict ?? true,
2361
                    },
2362
                };
2363
            }
2364
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
2365
            apiUrl = API_FIREWORKS;
2366
            apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS, request.body.secret_id);
2367
            headers = {};
2368
            bodyParams = {};
2369
            if (request.body.json_schema) {
2370
                bodyParams['response_format'] = {
2371
                    type: 'json_schema',
2372
                    json_schema: {
2373
                        name: request.body.json_schema.name,
2374
                        description: request.body.json_schema.description,
2375
                        schema: request.body.json_schema.value,
2376
                        strict: request.body.json_schema.strict ?? true,
2377
                    },
2378
                };
2379
            }
2380
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
2381
            apiUrl = API_NANOGPT;
2382
            apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT, request.body.secret_id);
2383
            headers = {};
2384
            bodyParams = {};
2385
            if (request.body.nanogpt_provider) {
2386
                headers['X-Provider'] = request.body.nanogpt_provider;
2387
            }
2388
            if (request.body.nanogpt_payg_override) {
2389
                headers['X-Billing-Mode'] = 'paygo';
2390
                bodyParams['billing_mode'] = 'paygo';
2391
            }
2392
            if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
2393
                request.body.model = `${request.body.model}:online`;
2394
            }
2395
            if (request.body.min_p !== undefined) {
2396
                bodyParams['min_p'] = request.body.min_p;
2397
            }
2398
            if (request.body.top_a !== undefined) {
2399
                bodyParams['top_a'] = request.body.top_a;
2400
            }
2401
            if (request.body.repetition_penalty !== undefined) {
2402
                bodyParams['repetition_penalty'] = request.body.repetition_penalty;
2403
            }
2404
            if (request.body.reasoning_effort) {
2405
                const effort = NANOGPT_REASONING_EFFORT_MAP[request.body.reasoning_effort];
2406
                bodyParams['reasoning'] = { effort: effort };
2407
            }
2408

2409
            const isClaude = /(?:^|\/)claude[-_]/.test(request.body.model);
2410
            if (enableSystemPromptCache && isClaude) {
2411
                bodyParams['cache_control'] = {
2412
                    'enabled': true,
2413
                    'ttl': cacheTTL,
2414
                };
2415
            }
2416
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
2417
            apiUrl = API_POLLINATIONS;
2418
            apiKey = readSecret(request.user.directories, SECRET_KEYS.POLLINATIONS, request.body.secret_id);
2419
            headers = {};
2420
            bodyParams = {
2421
                reasoning_effort: request.body.reasoning_effort,
2422
                seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
2423
            };
2424
            if (request.body.json_schema) {
2425
                bodyParams['response_format'] = {
2426
                    type: 'json_schema',
2427
                    json_schema: {
2428
                        schema: request.body.json_schema.value,
2429
                    },
2430
                };
2431
            }
2432
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
2433
            apiUrl = new URL(request.body.reverse_proxy || API_MOONSHOT).toString();
2434
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MOONSHOT, request.body.secret_id);
2435
            headers = {};
2436
            bodyParams = {
2437
                thinking: {
2438
                    type: request.body.include_reasoning ? 'enabled' : 'disabled',
2439
                },
2440
            };
2441
            request.body.json_schema
2442
                ? setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema)
2443
                : addAssistantPrefix(request.body.messages, [], 'partial');
2444
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
2445
            apiUrl = API_COMETAPI;
2446
            apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI, request.body.secret_id);
2447
            headers = {};
2448
            bodyParams = {
2449
                reasoning_effort: request.body.reasoning_effort,
2450
            };
2451
            throw new Error('This provider is temporarily disabled.');
2452
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZAI) {
2453
            const defaultApiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING ? API_ZAI_CODING : API_ZAI_COMMON;
2454
            apiUrl = new URL(request.body.reverse_proxy || defaultApiUrl).toString();
2455
            apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.ZAI, request.body.secret_id);
2456
            headers = {
2457
                'Accept-Language': 'en-US,en',
2458
            };
2459
            bodyParams = {
2460
                thinking: {
2461
                    type: request.body.include_reasoning ? 'enabled' : 'disabled',
2462
                },
2463
            };
2464
            if (request.body.json_schema) {
2465
                setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2466
            }
2467
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
2468
            const defaultApiUrl = request.body.siliconflow_endpoint === SILICONFLOW_ENDPOINT.CN
2469
                ? API_SILICONFLOW_CN : API_SILICONFLOW;
2470
            apiUrl = defaultApiUrl;
2471
            apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW, request.body.secret_id);
2472
            headers = {};
2473
            bodyParams = {};
2474
            if (request.body.json_schema) {
2475
                setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2476
            }
2477
        } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.WORKERS_AI) {
2478
            apiKey = readSecret(request.user.directories, SECRET_KEYS.WORKERS_AI, request.body.secret_id);
2479
            const accountId = String(request.body.workers_ai_account_id || '').trim();
2480
            if (!accountId) {
2481
                console.warn('Cloudflare Workers AI Account ID is missing.');
2482
                return response.status(400).send({ error: true });
2483
            }
2484
            apiUrl = `${API_WORKERS_AI}/${encodeURIComponent(accountId)}/ai/v1`;
2485
            headers = {};
2486
            bodyParams = {
2487
                repetition_penalty: request.body.repetition_penalty,
2488
            };
2489
            if (request.body.json_schema) {
2490
                bodyParams['response_format'] = {
2491
                    type: 'json_schema',
2492
                    json_schema: request.body.json_schema.value,
2493
                };
2494
            }
2495
        } else {
2496
            console.warn('This chat completion source is not supported yet.');
2497
            return response.status(400).send({ error: true });
2498
        }
2499

2500
        // A few of OpenAIs reasoning models support reasoning effort
2501
        if (request.body.reasoning_effort && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2502
            if (OPENAI_REASONING_EFFORT_MODELS.includes(request.body.model)) {
2503
                bodyParams['reasoning_effort'] = OPENAI_FIXED_REASONING_EFFORT[request.body.model] ?? OPENAI_REASONING_EFFORT_MAP[request.body.reasoning_effort] ?? request.body.reasoning_effort;
2504
            }
2505
            if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM && /^koboldcpp\/(.+)$/.test(request.body.model)) {
2506
                bodyParams['reasoning_effort'] = request.body.reasoning_effort;
2507
            }
2508
        }
2509

2510
        if (request.body.verbosity && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2511
            if (OPENAI_VERBOSITY_MODELS.test(request.body.model)) {
2512
                bodyParams['verbosity'] = request.body.verbosity;
2513
            }
2514
        }
2515

2516
        if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
2517
            console.warn('OpenAI API key is missing.');
2518
            return response.status(400).send({ error: true });
2519
        }
2520

2521
        // Add custom stop sequences
2522
        if (Array.isArray(request.body.stop) && request.body.stop.length > 0) {
2523
            bodyParams['stop'] = request.body.stop;
2524
        }
2525

2526
        const textPrompt = isTextCompletion ? convertTextCompletionPrompt(request.body.messages) : '';
2527
        const endpointUrl = isTextCompletion && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER ?
2528
            `${apiUrl}/completions` :
2529
            `${apiUrl}/chat/completions`;
2530

2531
        const controller = new AbortController();
2532
        request.socket.removeAllListeners('close');
2533
        request.socket.on('close', function () {
2534
            controller.abort();
2535
        });
2536

2537
        if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
2538
            bodyParams['tools'] = request.body.tools;
2539
            bodyParams['tool_choice'] = request.body.tool_choice;
2540
        }
2541

2542
        if (request.body.json_schema && !bodyParams['response_format']) {
2543
            bodyParams['response_format'] = {
2544
                type: 'json_schema',
2545
                json_schema: {
2546
                    name: request.body.json_schema.name,
2547
                    strict: request.body.json_schema.strict ?? true,
2548
                    schema: request.body.json_schema.value,
2549
                },
2550
            };
2551
        }
2552

2553
        const requestBody = {
2554
            'messages': isTextCompletion === false ? request.body.messages : undefined,
2555
            'prompt': isTextCompletion === true ? textPrompt : undefined,
2556
            'model': request.body.model,
2557
            'temperature': request.body.temperature,
2558
            'max_tokens': request.body.max_tokens,
2559
            'max_completion_tokens': request.body.max_completion_tokens,
2560
            'stream': request.body.stream,
2561
            'presence_penalty': request.body.presence_penalty,
2562
            'frequency_penalty': request.body.frequency_penalty,
2563
            'top_p': request.body.top_p,
2564
            'top_k': request.body.top_k,
2565
            'stop': isTextCompletion === false ? request.body.stop : undefined,
2566
            'logit_bias': request.body.logit_bias,
2567
            'seed': request.body.seed,
2568
            'n': request.body.n,
2569
            ...bodyParams,
2570
        };
2571

2572
        if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2573
            excludeKeysByYaml(requestBody, request.body.custom_exclude_body);
2574
        }
2575

2576
        /** @type {import('node-fetch').RequestInit} */
2577
        const config = {
2578
            method: 'post',
2579
            headers: {
2580
                'Content-Type': 'application/json',
2581
                'Authorization': 'Bearer ' + apiKey,
2582
                ...headers,
2583
            },
2584
            body: JSON.stringify(requestBody),
2585
            signal: controller.signal,
2586
        };
2587

2588
        console.debug('Chat Completion request:', requestBody);
2589

2590
        const fetchResponse = await fetch(endpointUrl, config);
2591

2592
        if (request.body.stream) {
2593
            console.info('Streaming request in progress');
2594
            return await forwardFetchResponse(fetchResponse, response);
2595
        }
2596

2597
        if (fetchResponse.ok) {
2598
            /** @type {any} */
2599
            const json = await fetchResponse.json();
2600
            console.debug('Chat Completion response:', json);
2601
            return response.send(json);
2602
        } else {
2603
            const responseText = await fetchResponse.text();
2604
            const errorData = tryParse(responseText);
2605

2606
            const message = fetchResponse.statusText || 'Unknown error occurred';
2607
            const quota_error = fetchResponse.status === 429 && errorData?.error?.type === 'insufficient_quota';
2608
            console.error('Chat completion request error: ', message, responseText);
2609

2610
            if (!response.headersSent) {
2611
                response.send({ error: { message }, quota_error: quota_error });
2612
            } else if (!response.writableEnded) {
2613
                response.write(responseText);
2614
            } else {
2615
                response.end();
2616
            }
2617
        }
2618
    } catch (error) {
2619
        console.error('Generation failed', error);
2620
        const message = error.code === 'ECONNREFUSED'
2621
            ? `Connection refused: ${error.message}`
2622
            : error.message || 'Unknown error occurred';
2623

2624
        if (!response.headersSent) {
2625
            response.status(502).send({ error: { message, ...error } });
2626
        } else {
2627
            response.end();
2628
        }
2629
    }
2630
});
2631

2632
const multimodalModels = express.Router();
2633

2634
multimodalModels.post('/pollinations', async (_req, res) => {
2635
    try {
2636
        const response = await fetch('https://gen.pollinations.ai/models');
2637

2638
        if (!response.ok) {
2639
            return res.json([]);
2640
        }
2641

2642
        /** @type {any} */
2643
        const data = await response.json();
2644

2645
        if (!Array.isArray(data)) {
2646
            return res.json([]);
2647
        }
2648

2649
        const multimodalModels = data
2650
            .filter(m => Array.isArray(m?.input_modalities))
2651
            .filter(m => m.input_modalities.includes('image'))
2652
            .map(m => m.name);
2653
        return res.json(multimodalModels);
2654
    } catch (error) {
2655
        console.error(error);
2656
        return res.sendStatus(500);
2657
    }
2658
});
2659

2660
multimodalModels.post('/aimlapi', async (_req, res) => {
2661
    try {
2662
        const response = await fetch('https://api.aimlapi.com/v1/models');
2663

2664
        if (!response.ok) {
2665
            return res.json([]);
2666
        }
2667

2668
        /** @type {any} */
2669
        const data = await response.json();
2670

2671
        if (!Array.isArray(data?.data)) {
2672
            return res.json([]);
2673
        }
2674

2675
        const multimodalModels = data.data.filter(m => m?.features?.includes('openai/chat-completion.vision')).map(m => m.id);
2676
        return res.json(multimodalModels);
2677
    } catch (error) {
2678
        console.error(error);
2679
        return res.sendStatus(500);
2680
    }
2681
});
2682

2683
multimodalModels.post('/nanogpt', async (_req, res) => {
2684
    try {
2685
        const response = await fetch('https://nano-gpt.com/api/v1/models?detailed=true');
2686

2687
        if (!response.ok) {
2688
            return res.json([]);
2689
        }
2690

2691
        /** @type {any} */
2692
        const data = await response.json();
2693

2694
        if (!Array.isArray(data?.data)) {
2695
            return res.json([]);
2696
        }
2697

2698
        const multimodalModels = data.data.filter(m => m?.capabilities?.vision).map(m => m.id);
2699
        return res.json(multimodalModels);
2700
    } catch (error) {
2701
        console.error(error);
2702
        return res.sendStatus(500);
2703
    }
2704
});
2705

2706
multimodalModels.post('/electronhub', async (_req, res) => {
2707
    try {
2708
        const response = await fetch('https://api.electronhub.ai/v1/models');
2709

2710
        if (!response.ok) {
2711
            return res.json([]);
2712
        }
2713

2714
        /** @type {any} */
2715
        const data = await response.json();
2716
        const multimodalModels = data.data.filter(m => m.metadata?.vision).map(m => m.id);
2717
        return res.json(multimodalModels);
2718
    } catch (error) {
2719
        console.error(error);
2720
        return res.sendStatus(500);
2721
    }
2722
});
2723

2724
multimodalModels.post('/chutes', async (req, res) => {
2725
    try {
2726
        const key = readSecret(req.user.directories, SECRET_KEYS.CHUTES);
2727

2728
        if (!key) {
2729
            return res.json([]);
2730
        }
2731

2732
        const response = await fetch('https://llm.chutes.ai/v1/models', {
2733
            headers: {
2734
                'Authorization': `Bearer ${key}`,
2735
            },
2736
        });
2737

2738
        if (!response.ok) {
2739
            return res.json([]);
2740
        }
2741

2742
        const data = await response.json();
2743

2744
        const modelsData = /** @type {{object: string, data: Array<{id: string, input_modalities?: string[]}>}} */ (data);
2745
        const multimodalModels = modelsData.data
2746
            .filter(m => m.input_modalities?.includes('image'))
2747
            .map(m => m.id);
2748
        return res.json(multimodalModels);
2749
    } catch (error) {
2750
        console.error(error);
2751
        return res.sendStatus(500);
2752
    }
2753
});
2754

2755
multimodalModels.post('/mistral', async (req, res) => {
2756
    try {
2757
        const key = readSecret(req.user.directories, SECRET_KEYS.MISTRALAI);
2758

2759
        if (!key) {
2760
            return res.json([]);
2761
        }
2762

2763
        const response = await fetch('https://api.mistral.ai/v1/models', {
2764
            headers: {
2765
                'Authorization': `Bearer ${key}`,
2766
            },
2767
        });
2768

2769
        if (!response.ok) {
2770
            return res.json([]);
2771
        }
2772

2773
        /** @type {any} */
2774
        const data = await response.json();
2775
        const multimodalModels = data.data.filter(m => m.capabilities?.vision).map(m => m.id);
2776
        return res.json(multimodalModels);
2777
    } catch (error) {
2778
        console.error(error);
2779
        return res.sendStatus(500);
2780
    }
2781
});
2782

2783
multimodalModels.post('/xai', async (req, res) => {
2784
    try {
2785
        const key = readSecret(req.user.directories, SECRET_KEYS.XAI);
2786

2787
        if (!key) {
2788
            return res.json([]);
2789
        }
2790

2791
        // xAI's /models endpoint doesn't return modality info, so we must use /language-models instead
2792
        const response = await fetch('https://api.x.ai/v1/language-models', {
2793
            headers: {
2794
                'Authorization': `Bearer ${key}`,
2795
            },
2796
        });
2797

2798
        if (!response.ok) {
2799
            return res.json([]);
2800
        }
2801

2802
        /** @type {any} */
2803
        const data = await response.json();
2804
        const multimodalModels = data.models.filter(m => m.input_modalities?.includes('image')).map(m => m.id);
2805
        if (!multimodalModels.includes('grok-4-0709')) {
2806
            // The endpoint says it doesn't support images, but it does
2807
            multimodalModels.push('grok-4-0709');
2808
        }
2809
        return res.json(multimodalModels);
2810
    } catch (error) {
2811
        console.error(error);
2812
        return res.sendStatus(500);
2813
    }
2814
});
2815

2816
multimodalModels.post('/moonshot', async (req, res) => {
2817
    try {
2818
        const key = readSecret(req.user.directories, SECRET_KEYS.MOONSHOT);
2819

2820
        if (!key) {
2821
            return res.json([]);
2822
        }
2823

2824
        const response = await fetch('https://api.moonshot.ai/v1/models', {
2825
            headers: {
2826
                'Authorization': `Bearer ${key}`,
2827
            },
2828
        });
2829

2830
        if (!response.ok) {
2831
            return res.json([]);
2832
        }
2833

2834
        /** @type {any} */
2835
        const data = await response.json();
2836

2837
        const multimodalModels = data.data.filter(m => m.supports_image_in).map(m => m.id);
2838
        return res.json(multimodalModels);
2839
    } catch (error) {
2840
        console.error(error);
2841
        return res.sendStatus(500);
2842
    }
2843
});
2844

2845
multimodalModels.post('/workers_ai', async (req, res) => {
2846
    try {
2847
        const key = readSecret(req.user.directories, SECRET_KEYS.WORKERS_AI);
2848
        const accountId = String(req.body.workers_ai_account_id || '').trim();
2849

2850
        if (!key || !accountId) {
2851
            return res.json([]);
2852
        }
2853

2854
        const apiUrl = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/models/search?task=Text+Generation&per_page=1000`;
2855
        const response = await fetch(apiUrl, {
2856
            method: 'GET',
2857
            headers: { 'Authorization': 'Bearer ' + key },
2858
        });
2859

2860
        if (!response.ok) {
2861
            return res.json([]);
2862
        }
2863

2864
        /** @type {any} */
2865
        const data = await response.json();
2866
        const models = Array.isArray(data?.result)
2867
            ? data.result
2868
                .filter(m => Array.isArray(m.properties) && m.properties.some(p => p.property_id === 'vision' && p.value === 'true'))
2869
                .map(m => m.name)
2870
            : [];
2871
        return res.json(models);
2872
    } catch (error) {
2873
        console.error(error);
2874
        return res.sendStatus(500);
2875
    }
2876
});
2877

2878
router.use('/multimodal-models', multimodalModels);
2879

2880
router.post('/process', async function (request, response) {
2881
    try {
2882
        if (!Array.isArray(request.body.messages)) {
2883
            return response.status(400).send({ error: 'Invalid messages format' });
2884
        }
2885

2886
        if (!Object.values(PROMPT_PROCESSING_TYPE).includes(request.body.type)) {
2887
            return response.status(400).send({ error: 'Unknown processing type' });
2888
        }
2889

2890
        const messages = postProcessPrompt(request.body.messages, request.body.type, getPromptNames(request));
2891
        return response.send({ messages });
2892
    } catch (error) {
2893
        console.error(error);
2894
        return response.sendStatus(500);
2895
    }
2896
});