Refactor CC API async route handlers (#4885) * Improve error handling in CC /status and /generate endpoints * Cancel pending status check on switching CC source

9046fe8d2de22b166e588839c8579c9b1da754c0

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 10a2f0f7406035ca064eecc2e85f52a707ae770b
3 files changed, +539 -557Ignore whitespace
public/script.js+1 -1
@@ -744,7 +744,7 @@ function initStandaloneMode() {
744744 }
745745}
746746
747747export function cancelStatusCheck(reason = 'Manually cancelled status check') {
748748 abortStatusCheck?.abort(new AbortReason(reason));
749749 abortStatusCheck = new AbortController();
750750 setOnlineStatus('no_connection');
public/scripts/openai.js+2 -0
@@ -7,6 +7,7 @@ import { Fuse, DOMPurify } from '../lib.js';
77
88import {
99 abortStatusCheck,
10+ cancelStatusCheck,
1011 characters,
1112 event_types,
1213 eventSource,
@@ -6349,6 +6350,7 @@ export function initOpenAI() {
63496350 });
63506351
63516352 $('#chat_completion_source').on('change', function () {
6353+ cancelStatusCheck('Chat Completion source changed');
63526354 model_list = [];
63536355 oai_settings.chat_completion_source = String($(this).find(':selected').val());
63546356 toggleChatCompletionForms();
src/endpoints/backends/chat-completions.js+536 -556
@@ -1560,210 +1560,210 @@ async function sendAzureOpenAIRequest(request, response) {
15601560export const router = express.Router();
15611561
15621562router.post('/status', async function (request, statusResponse) {
1563- if (!request.body) return statusResponse.sendStatus(400);
1563+ try {
1564-
1564+ if (!request.body) return statusResponse.sendStatus(400);
1565- let apiUrl = '';
1565+
15661566 let apiKeyapiUrl = '';
15671567 let headersapiKey = {}'';
15681568 let queryParamsheaders = {};
1569-
1569+ let queryParams = {};
1570- if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1570+
1571- apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1571+ if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1572- apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1572+ apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1573- headers = {};
1573+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1574- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1574+ headers = {};
1575- apiUrl = 'https://openrouter.ai/api/v1';
1575+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1576- apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1576+ apiUrl = 'https://openrouter.ai/api/v1';
1577- // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1577+ apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1578- headers = { ...OPENROUTER_HEADERS };
1578+ // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1579- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
1579+ headers = { ...OPENROUTER_HEADERS };
15801580 apiUrl} =else newif URL(request.body.reverse_proxychat_completion_source ||=== API_MISTRAL)CHAT_COMPLETION_SOURCES.toString(MISTRALAI); {
1581- apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);
1581+ apiUrl = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
1582- headers = {};
1582+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);
1583- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
1583+ headers = {};
1584- apiUrl = request.body.custom_url;
1584+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
1585- apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
1585+ apiUrl = request.body.custom_url;
1586- headers = {};
1586+ apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
1587- mergeObjectWithYaml(headers, request.body.custom_include_headers);
1587+ headers = {};
1588- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {
1588+ mergeObjectWithYaml(headers, request.body.custom_include_headers);
1589- apiUrl = API_COHERE_V1;
1589+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {
1590- apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE);
1590+ apiUrl = API_COHERE_V1;
1591- headers = {};
1591+ apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE);
1592- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CHUTES) {
1592+ headers = {};
1593- apiUrl = API_CHUTES;
1593+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CHUTES) {
1594- apiKey = readSecret(request.user.directories, SECRET_KEYS.CHUTES);
1594+ apiUrl = API_CHUTES;
1595- headers = {};
1595+ apiKey = readSecret(request.user.directories, SECRET_KEYS.CHUTES);
1596- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ELECTRONHUB) {
1596+ headers = {};
1597- apiUrl = API_ELECTRONHUB;
1597+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ELECTRONHUB) {
1598- apiKey = readSecret(request.user.directories, SECRET_KEYS.ELECTRONHUB);
1598+ apiUrl = API_ELECTRONHUB;
1599- headers = {};
1599+ apiKey = readSecret(request.user.directories, SECRET_KEYS.ELECTRONHUB);
1600- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
1600+ headers = {};
1601- apiUrl = API_NANOGPT;
1601+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
1602- apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
1602+ apiUrl = API_NANOGPT;
1603- headers = {};
1603+ apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
16041604 queryParams headers = { detailed: true };
1605- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {
1605+ queryParams = { detailed: true };
1606- apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK.replace('/beta', '')).toString();
1606+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {
1607- apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
1607+ apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK.replace('/beta', '')).toString();
1608- headers = {};
1608+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
1609- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.XAI) {
1609+ headers = {};
16101610 apiUrl} =else newif URL(request.body.reverse_proxychat_completion_source ||=== API_XAI)CHAT_COMPLETION_SOURCES.toString(XAI); {
1611- apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.XAI);
1611+ apiUrl = new URL(request.body.reverse_proxy || API_XAI).toString();
1612- headers = {};
1612+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.XAI);
1613- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AIMLAPI) {
1613+ headers = {};
1614- apiUrl = API_AIMLAPI;
1614+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AIMLAPI) {
1615- apiKey = readSecret(request.user.directories, SECRET_KEYS.AIMLAPI);
1615+ apiUrl = API_AIMLAPI;
1616- headers = { ...AIMLAPI_HEADERS };
1616+ apiKey = readSecret(request.user.directories, SECRET_KEYS.AIMLAPI);
1617- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
1617+ headers = { ...AIMLAPI_HEADERS };
1618- apiUrl = 'https://text.pollinations.ai';
1618+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
1619- apiKey = 'NONE';
1619+ apiUrl = 'https://text.pollinations.ai';
1620- headers = {};
1620+ apiKey = 'NONE';
1621- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
1621+ headers = {};
1622- apiUrl = API_GROQ;
1622+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
1623- apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
1623+ apiUrl = API_GROQ;
1624- headers = {};
1624+ apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
1625- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
1625+ headers = {};
1626- apiUrl = API_COMETAPI;
1626+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
1627- apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
1627+ apiUrl = API_COMETAPI;
1628- headers = {};
1628+ apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
1629- throw new Error('This provider is temporarily disabled.');
1629+ headers = {};
1630- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
1630+ throw new Error('This provider is temporarily disabled.');
1631- apiUrl = API_MOONSHOT;
1631+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
1632- apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
1632+ apiUrl = API_MOONSHOT;
1633- headers = {};
1633+ apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
1634- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
1634+ headers = {};
1635- apiUrl = API_FIREWORKS;
1635+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
1636- apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
1636+ apiUrl = API_FIREWORKS;
1637- headers = {};
1637+ apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
1638- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MAKERSUITE) {
1638+ headers = {};
1639- apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
1639+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MAKERSUITE) {
1640- apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
1640+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
1641- const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
1641+ apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
1642- const modelsUrl = !apiKey && request.body.reverse_proxy
1642+ const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
1643- ? `${apiUrl}/${apiVersion}/models`
1643+ const modelsUrl = !apiKey && request.body.reverse_proxy
16441644 : ? `${apiUrl}/${apiVersion}/models?key=${apiKey}`;
1645-
1645+ : `${apiUrl}/${apiVersion}/models?key=${apiKey}`;
1646- if (!apiKey && !request.body.reverse_proxy) {
1646+
1647- console.warn('Google AI Studio API key is missing.');
1647+ if (!apiKey && !request.body.reverse_proxy) {
1648- return statusResponse.status(400).send({ error: true });
1648+ console.warn('Google AI Studio API key is missing.');
1649- }
1649+ return statusResponse.status(400).send({ error: true });
1650+ }
16501651
16511652 try {
16521653 const response = await fetch(modelsUrl);
16531654
16541655 if (response.ok) {
16551656 /** @type {any} */
16561657 const data = await response.json();
16571658 // Transform Google AI Studio models to OpenAI format
16581659 const models = data.models
16591660 ?.filter(model => model.supportedGenerationMethods?.includes('generateContent'))
16601661 ?.map(model => ({
16611662 id: model.name.replace('models/', ''),
16621663 })) || [];
16631664
16641665 console.info('Available Google AI Studio models:', models.map(m => m.id));
16651666 return statusResponse.send({ data: models });
16661667 } else {
16671668 console.warn('Google AI Studio models endpoint failed:', response.status, response.statusText);
1669+ return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
1670+ }
1671+ } catch (error) {
1672+ console.error('Error fetching Google AI Studio models:', error);
16681673 return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
16691674 }
1670- } catch (error) {
1675+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AZURE_OPENAI) {
1671- console.error('Error fetching Google AI Studio models:', error);
1676+ const { azure_base_url, azure_deployment_name, azure_api_version } = request.body;
1672- return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
1677+ const apiKey = readSecret(request.user.directories, SECRET_KEYS.AZURE_OPENAI);
1673- }
1678+
1674- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AZURE_OPENAI) {
1679+ // 1) Validate configuration from the frontend
1675- const { azure_base_url, azure_deployment_name, azure_api_version } = request.body;
1680+ if (!apiKey || !azure_base_url || !azure_deployment_name || !azure_api_version) {
1676- const apiKey = readSecret(request.user.directories, SECRET_KEYS.AZURE_OPENAI);
1681+ console.warn('Azure OpenAI status check failed: missing config from frontend.');
1677-
1682+ return statusResponse.status(400).send({ error: true, message: 'Azure configuration is incomplete.' });
1678- // 1) Validate configuration from the frontend
1683+ }
1679- if (!apiKey || !azure_base_url || !azure_deployment_name || !azure_api_version) {
1684+ // 2) Build URLs using the URL API for consistency and robustness.
1680- console.warn('Azure OpenAI status check failed: missing config from frontend.');
1685+ const modelsUrl = new URL('/openai/models', azure_base_url);
1681- return statusResponse.status(400).send({ error: true, message: 'Azure configuration is incomplete.' });
1686+ modelsUrl.searchParams.set('api-version', azure_api_version);
1682- }
1687+
1683- // 2) Build URLs using the URL API for consistency and robustness.
1688+ const chatUrl = new URL(`/openai/deployments/${azure_deployment_name}/chat/completions`, azure_base_url);
1684- const modelsUrl = new URL('/openai/models', azure_base_url);
1689+ chatUrl.searchParams.set('api-version', azure_api_version);
1685- modelsUrl.searchParams.set('api-version', azure_api_version);
1690+
1686-
1691+ // Map common status codes to user-friendly error messages
1687- const chatUrl = new URL(`/openai/deployments/${azure_deployment_name}/chat/completions`, azure_base_url);
1692+ const azureStatusErrorMap = {
1688- chatUrl.searchParams.set('api-version', azure_api_version);
1693+ 400: 'API version may be invalid for this resource.',
1689-
1694+ 401: 'Invalid API key or insufficient permissions.',
1690- // Map common status codes to user-friendly error messages
1695+ 403: 'Invalid API key or insufficient permissions.',
1691- const azureStatusErrorMap = {
1696+ 404: 'Endpoint URL appears incorrect (404).',
1692- 400: 'API version may be invalid for this resource.',
1697+ };
1693- 401: 'Invalid API key or insufficient permissions.',
1694- 403: 'Invalid API key or insufficient permissions.',
1695- 404: 'Endpoint URL appears incorrect (404).',
1696- };
16971698
16981699 try {
16991700 // ---- A) GET /models: fast sanity check for endpoint + api key + api version ----
17001701 const apiConfigTest = await fetch(modelsUrl, {
17011702 method: 'GET',
17021703 headers: { 'api-key': apiKey, 'Accept': 'application/json' },
17031704 });
17041705
17051706 if (!apiConfigTest.ok) {
17061707 let errText = '';
17071708 try { errText = await apiConfigTest.text(); } catch { /* response body may be empty */ }
17081709
17091710 console.warn('Azure OpenAI GET /models failed:', apiConfigTest.status, apiConfigTest.statusText, errText || '');
17101711
17111712 const defaultMessage = `Azure Models endpoint error: ${apiConfigTest.statusText}`;
17121713 const message = azureStatusErrorMap[apiConfigTest.status] ?? defaultMessage;
17131714 return statusResponse.status(apiConfigTest.status).send({ error: true, message });
17141715 }
17151716
17161717 // ---- B) POST /chat/completions: verify deployment + read underlying model ID ----
17171718 // Small, deterministic probe to minimize cost/latency
17181719 const modelPayload = {
17191720 messages: [{ role: 'user', content: 'Say word Hi' }],
17201721 stream: false,
17211722 max_completion_tokens: 5,
17221723 };
17231724
17241725 const modelRequest = await fetch(chatUrl, {
17251726 method: 'POST',
17261727 headers: { 'api-key': apiKey, 'Content-Type': 'application/json', 'Accept': 'application/json' },
17271728 body: JSON.stringify(modelPayload),
17281729 });
17291730
17301731 let modelResponse;
17311732 try {
17321733 modelResponse = await modelRequest.json();
17331734 } catch {
17341735 modelResponse = { raw: 'Failed to parse JSON response from chat completions probe.' };
17351736 }
17361737
17371738 const modelId = /** @type {any} */ (modelResponse)?.model;
17381739 if (!modelId) {
17391740 console.warn('Azure status check succeeded but could not find a model ID in the response.');
17401741 console.debug('Azure Response Body:', modelResponse);
17411742 // Keep a benign success to avoid UX disruption in the UI
17421743 return statusResponse.send({ data: [] });
17431744 }
17441745
17451746 console.info(color.green('Azure OpenAI connection successful. Detected model:'), modelId);
17461747 // Consistent response format: always an array of { id }
17471748 return statusResponse.send({ data: [{ id: modelId }] });
17481749 } catch (error) {
17491750 console.error('Azure OpenAI status check connection error:', error);
17501751 return statusResponse.status(500).send({ error: true, message: 'Failed to connect to the Azure endpoint.' });
1752+ }
1753+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
1754+ apiUrl = API_SILICONFLOW;
1755+ apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
1756+ headers = {};
1757+ } else {
1758+ console.warn('This chat completion source is not supported yet.');
1759+ return statusResponse.status(400).send({ error: true });
17511760 }
1752- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
1753- apiUrl = API_SILICONFLOW;
1754- apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
1755- headers = {};
1756- } else {
1757- console.warn('This chat completion source is not supported yet.');
1758- return statusResponse.status(400).send({ error: true });
1759- }
17601761
17611762 if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
17621763 console.warn('Chat Completion API key is missing.');
17631764 return statusResponse.status(400).send({ error: true });
17641765 }
17651766
1766- try {
17671767 const modelsUrl = new URL(urlJoin(apiUrl, '/models'));
17681768 Object.keys(queryParams).forEach(key => {
17691769 modelsUrl.searchParams.append(key, queryParams[key]);
@@ -1933,429 +1933,409 @@ router.post('/bias', async function (request, response) {
19331933 }
19341934});
19351935
1936+router.post('/generate', async function (request, response) {
1937+ try {
1938+ if (!request.body) return response.status(400).send({ error: true });
1939+
1940+ const postProcessingType = request.body.custom_prompt_post_processing;
1941+ if (Array.isArray(request.body.messages) && postProcessingType) {
1942+ console.info('Applying custom prompt post-processing of type', postProcessingType);
1943+ request.body.messages = postProcessPrompt(
1944+ request.body.messages,
1945+ postProcessingType,
1946+ getPromptNames(request));
1947+ }
1948+
1949+ if (request.body.json_schema?.value) {
1950+ request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
1951+ }
1952+
1953+ switch (request.body.chat_completion_source) {
1954+ case CHAT_COMPLETION_SOURCES.CLAUDE: return await sendClaudeRequest(request, response);
1955+ case CHAT_COMPLETION_SOURCES.AI21: return await sendAI21Request(request, response);
1956+ case CHAT_COMPLETION_SOURCES.MAKERSUITE: return await sendMakerSuiteRequest(request, response);
1957+ case CHAT_COMPLETION_SOURCES.VERTEXAI: return await sendMakerSuiteRequest(request, response);
1958+ case CHAT_COMPLETION_SOURCES.MISTRALAI: return await sendMistralAIRequest(request, response);
1959+ case CHAT_COMPLETION_SOURCES.COHERE: return await sendCohereRequest(request, response);
1960+ case CHAT_COMPLETION_SOURCES.DEEPSEEK: return await sendDeepSeekRequest(request, response);
1961+ case CHAT_COMPLETION_SOURCES.AIMLAPI: return await sendAimlapiRequest(request, response);
1962+ case CHAT_COMPLETION_SOURCES.XAI: return await sendXaiRequest(request, response);
1963+ case CHAT_COMPLETION_SOURCES.CHUTES: return await sendChutesRequest(request, response);
1964+ case CHAT_COMPLETION_SOURCES.ELECTRONHUB: return await sendElectronHubRequest(request, response);
1965+ case CHAT_COMPLETION_SOURCES.AZURE_OPENAI: return await sendAzureOpenAIRequest(request, response);
1966+ }
1967+
1968+ let apiUrl;
1969+ let apiKey;
1970+ let headers;
1971+ let bodyParams;
1972+ const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model)) || typeof request.body.messages === 'string';
1973+
1974+ if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1975+ apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1976+ apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1977+ headers = {};
1978+ bodyParams = {
1979+ logprobs: request.body.logprobs,
1980+ top_logprobs: undefined,
1981+ };
19361982
1937-router.post('/generate', function (request, response) {
1983+ // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
1938- if (!request.body) return response.status(400).send({ error: true });
1984+ if (!isTextCompletion && bodyParams.logprobs > 0) {
1985+ bodyParams.top_logprobs = bodyParams.logprobs;
1986+ bodyParams.logprobs = true;
1987+ }
19391988
1940- const postProcessingType = request.body.custom_prompt_post_processing;
1989+ if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {
1941- if (Array.isArray(request.body.messages) && postProcessingType) {
1990+ bodyParams['user'] = uuidv4();
1942- console.info('Applying custom prompt post-processing of type', postProcessingType);
1991+ }
1943- request.body.messages = postProcessPrompt(
1992+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1944- request.body.messages,
1993+ apiUrl = 'https://openrouter.ai/api/v1';
1945- postProcessingType,
1994+ apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1946- getPromptNames(request));
1995+ // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1947- }
1996+ headers = { ...OPENROUTER_HEADERS };
1997+ bodyParams = {
1998+ 'transforms': getOpenRouterTransforms(request),
1999+ 'plugins': getOpenRouterPlugins(request),
2000+ 'include_reasoning': Boolean(request.body.include_reasoning),
2001+ };
19482002
19492003 if (request.body.json_schema?.valuemin_p !== undefined) {
1950- request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
2004+ bodyParams['min_p'] = request.body.min_p;
19512005 }
19522006
19532007 switch if (request.body.chat_completion_sourcetop_a !== undefined) {
1954- case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response);
2008+ bodyParams['top_a'] = request.body.top_a;
1955- case CHAT_COMPLETION_SOURCES.AI21: return sendAI21Request(request, response);
2009+ }
1956- case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response);
1957- case CHAT_COMPLETION_SOURCES.VERTEXAI: return sendMakerSuiteRequest(request, response);
1958- case CHAT_COMPLETION_SOURCES.MISTRALAI: return sendMistralAIRequest(request, response);
1959- case CHAT_COMPLETION_SOURCES.COHERE: return sendCohereRequest(request, response);
1960- case CHAT_COMPLETION_SOURCES.DEEPSEEK: return sendDeepSeekRequest(request, response);
1961- case CHAT_COMPLETION_SOURCES.AIMLAPI: return sendAimlapiRequest(request, response);
1962- case CHAT_COMPLETION_SOURCES.XAI: return sendXaiRequest(request, response);
1963- case CHAT_COMPLETION_SOURCES.CHUTES: return sendChutesRequest(request, response);
1964- case CHAT_COMPLETION_SOURCES.ELECTRONHUB: return sendElectronHubRequest(request, response);
1965- case CHAT_COMPLETION_SOURCES.AZURE_OPENAI: return sendAzureOpenAIRequest(request, response);
1966- }
19672010
1968- let apiUrl;
2011+ if (request.body.repetition_penalty !== undefined) {
1969- let apiKey;
2012+ bodyParams['repetition_penalty'] = request.body.repetition_penalty;
1970- let headers;
2013+ }
1971- let bodyParams;
1972- const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model)) || typeof request.body.messages === 'string';
1973-
1974- if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1975- apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1976- apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1977- headers = {};
1978- bodyParams = {
1979- logprobs: request.body.logprobs,
1980- top_logprobs: undefined,
1981- };
19822014
1983- // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
2015+ if (Array.isArray(request.body.provider) && request.body.provider.length > 0) {
1984- if (!isTextCompletion && bodyParams.logprobs > 0) {
2016+ bodyParams['provider'] = {
1985- bodyParams.top_logprobs = bodyParams.logprobs;
2017+ allow_fallbacks: request.body.allow_fallbacks ?? true,
1986- bodyParams.logprobs = true;
2018+ order: request.body.provider ?? [],
1987- }
2019+ };
2020+ }
19882021
1989- if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {
2022+ if (request.body.use_fallback) {
19902023 bodyParams['userroute'] = uuidv4()'fallback';
19912024 }
1992- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1993- apiUrl = 'https://openrouter.ai/api/v1';
1994- apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1995- // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1996- headers = { ...OPENROUTER_HEADERS };
1997- bodyParams = {
1998- 'transforms': getOpenRouterTransforms(request),
1999- 'plugins': getOpenRouterPlugins(request),
2000- 'include_reasoning': Boolean(request.body.include_reasoning),
2001- };
20022025
20032026 if (request.body.min_p !== undefinedreasoning_effort) {
20042027 bodyParams['min_preasoning'] = { effort: request.body.min_preasoning_effort };
20052028 }
20062029
20072030 if (request.body.top_a !== undefinedverbosity) {
20082031 bodyParams['top_averbosity'] = request.body.top_averbosity;
20092032 }
20102033
20112034 if (request.body.repetition_penalty !== undefinedjson_schema) {
20122035 bodyParams['repetition_penaltyresponse_format'] = request.body.repetition_penalty;{
2013- }
2036+ type: 'json_schema',
2037+ json_schema: {
2038+ name: request.body.json_schema.name,
2039+ strict: request.body.json_schema.strict ?? true,
2040+ schema: request.body.json_schema.value,
2041+ },
2042+ };
2043+ }
20142044
2015- if (Array.isArray(request.body.provider) && request.body.provider.length > 0) {
2045+ const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2016- bodyParams['provider'] = {
2046+ const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
2017- allow_fallbacks: request.body.allow_fallbacks ?? true,
2047+ const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2018- order: request.body.provider ?? [],
2048+ const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2019- };
2049+ if (Array.isArray(request.body.messages)) {
2020- }
2050+ embedOpenRouterMedia(request.body.messages);
20212051
20222052 if (request.body.use_fallbackisClaude3or4) {
2023- bodyParams['route'] = 'fallback';
2053+ if (enableSystemPromptCache) {
2024- }
2054+ cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
2055+ }
20252056
2026- if (request.body.reasoning_effort) {
2057+ if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
2027- bodyParams['reasoning'] = { effort: request.body.reasoning_effort };
2058+ cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
20282059 }
2060+ }
2061+ }
20292062
2030- if (request.body.verbosity) {
2063+ const isGemini = /google\/gemini/.test(request.body.model);
2031- bodyParams['verbosity'] = request.body.verbosity;
2064+ if (isGemini) {
2032- }
2065+ bodyParams['safety_settings'] = GEMINI_SAFETY;
2066+ }
2067+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2068+ apiUrl = request.body.custom_url;
2069+ apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
2070+ headers = {};
2071+ bodyParams = {
2072+ logprobs: request.body.logprobs,
2073+ top_logprobs: undefined,
2074+ };
20332075
2034- if (request.body.json_schema) {
2076+ // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
2035- bodyParams['response_format'] = {
2077+ if (!isTextCompletion && bodyParams.logprobs > 0) {
2036- type: 'json_schema',
2078+ bodyParams.top_logprobs = bodyParams.logprobs;
2037- json_schema: {
2079+ bodyParams.logprobs = true;
2038- name: request.body.json_schema.name,
2080+ }
2039- strict: request.body.json_schema.strict ?? true,
2081+
2040- schema: request.body.json_schema.value,
2082+ mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
2083+ mergeObjectWithYaml(headers, request.body.custom_include_headers);
2084+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
2085+ apiUrl = API_PERPLEXITY;
2086+ apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
2087+ headers = {};
2088+ bodyParams = {
2089+ reasoning_effort: request.body.reasoning_effort,
2090+ };
2091+ request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
2092+ if (request.body.json_schema) {
2093+ bodyParams['response_format'] = {
2094+ type: 'json_schema',
2095+ json_schema: {
2096+ schema: request.body.json_schema.value,
2097+ },
2098+ };
2099+ }
2100+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
2101+ apiUrl = API_GROQ;
2102+ apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
2103+ headers = {};
2104+ bodyParams = {};
2105+ if (request.body.json_schema) {
2106+ bodyParams['response_format'] = {
2107+ type: 'json_schema',
2108+ json_schema: {
2109+ name: request.body.json_schema.name,
2110+ description: request.body.json_schema.description,
2111+ schema: request.body.json_schema.value,
2112+ strict: request.body.json_schema.strict ?? true,
2113+ },
2114+ };
2115+ }
2116+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
2117+ apiUrl = API_FIREWORKS;
2118+ apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
2119+ headers = {};
2120+ bodyParams = {};
2121+ if (request.body.json_schema) {
2122+ bodyParams['response_format'] = {
2123+ type: 'json_schema',
2124+ json_schema: {
2125+ name: request.body.json_schema.name,
2126+ description: request.body.json_schema.description,
2127+ schema: request.body.json_schema.value,
2128+ strict: request.body.json_schema.strict ?? true,
2129+ },
2130+ };
2131+ }
2132+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
2133+ apiUrl = API_NANOGPT;
2134+ apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
2135+ headers = {};
2136+ bodyParams = {};
2137+ if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
2138+ request.body.model = `${request.body.model}:online`;
2139+ }
2140+ if (request.body.min_p !== undefined) {
2141+ bodyParams['min_p'] = request.body.min_p;
2142+ }
2143+ if (request.body.top_a !== undefined) {
2144+ bodyParams['top_a'] = request.body.top_a;
2145+ }
2146+ if (request.body.repetition_penalty !== undefined) {
2147+ bodyParams['repetition_penalty'] = request.body.repetition_penalty;
2148+ }
2149+ const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2150+ const isClaude3or4 = /claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2151+ const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2152+ if (enableSystemPromptCache && isClaude3or4) {
2153+ bodyParams['cache_control'] = {
2154+ 'enabled': true,
2155+ 'ttl': cacheTTL,
2156+ };
2157+ }
2158+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
2159+ apiUrl = API_POLLINATIONS;
2160+ apiKey = 'NONE';
2161+ headers = {
2162+ 'Authorization': '',
2163+ };
2164+ bodyParams = {
2165+ reasoning_effort: request.body.reasoning_effort,
2166+ private: true,
2167+ referrer: 'sillytavern',
2168+ seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
2169+ };
2170+ if (request.body.json_schema) {
2171+ setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2172+ }
2173+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
2174+ apiUrl = API_MOONSHOT;
2175+ apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
2176+ headers = {};
2177+ bodyParams = {};
2178+ request.body.json_schema
2179+ ? setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema)
2180+ : addAssistantPrefix(request.body.messages, [], 'partial');
2181+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
2182+ apiUrl = API_COMETAPI;
2183+ apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
2184+ headers = {};
2185+ bodyParams = {
2186+ reasoning_effort: request.body.reasoning_effort,
2187+ };
2188+ throw new Error('This provider is temporarily disabled.');
2189+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZAI) {
2190+ apiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING ? API_ZAI_CODING : API_ZAI_COMMON;
2191+ apiKey = readSecret(request.user.directories, SECRET_KEYS.ZAI);
2192+ headers = {
2193+ 'Accept-Language': 'en-US,en',
2194+ };
2195+ bodyParams = {
2196+ thinking: {
2197+ type: request.body.include_reasoning ? 'enabled' : 'disabled',
20412198 },
20422199 };
2200+ if (request.body.json_schema) {
2201+ setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2202+ }
2203+ } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
2204+ apiUrl = API_SILICONFLOW;
2205+ apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
2206+ headers = {};
2207+ bodyParams = {};
2208+ if (request.body.json_schema) {
2209+ setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2210+ }
2211+ } else {
2212+ console.warn('This chat completion source is not supported yet.');
2213+ return response.status(400).send({ error: true });
20432214 }
20442215
2045- const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2216+ // A few of OpenAIs reasoning models support reasoning effort
2046- const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
2217+ if (request.body.reasoning_effort && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2047- const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2218+ if (OPENAI_REASONING_EFFORT_MODELS.includes(request.body.model)) {
2048- const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2219+ bodyParams['reasoning_effort'] = OPENAI_REASONING_EFFORT_MAP[request.body.reasoning_effort] ?? request.body.reasoning_effort;
2049- if (Array.isArray(request.body.messages)) {
2220+ }
2050- embedOpenRouterMedia(request.body.messages);
2221+ }
2051-
2052- if (isClaude3or4) {
2053- if (enableSystemPromptCache) {
2054- cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
2055- }
20562222
2057- if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
2223+ if (request.body.verbosity && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2058- cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
2224+ if (OPENAI_VERBOSITY_MODELS.test(request.body.model)) {
2059- }
2225+ bodyParams['verbosity'] = request.body.verbosity;
20602226 }
20612227 }
20622228
2063- const isGemini = /google\/gemini/.test(request.body.model);
2229+ if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
2064- if (isGemini) {
2230+ console.warn('OpenAI API key is missing.');
2065- bodyParams['safety_settings'] = GEMINI_SAFETY;
2231+ return response.status(400).send({ error: true });
20662232 }
2067- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2068- apiUrl = request.body.custom_url;
2069- apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
2070- headers = {};
2071- bodyParams = {
2072- logprobs: request.body.logprobs,
2073- top_logprobs: undefined,
2074- };
20752233
2076- // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
2234+ // Add custom stop sequences
20772235 if (!isTextCompletionArray.isArray(request.body.stop) && bodyParamsrequest.logprobsbody.stop.length > 0) {
20782236 bodyParams.top_logprobs['stop'] = bodyParamsrequest.logprobsbody.stop;
2079- bodyParams.logprobs = true;
20802237 }
20812238
2082- mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
2239+ const textPrompt = isTextCompletion ? convertTextCompletionPrompt(request.body.messages) : '';
2083- mergeObjectWithYaml(headers, request.body.custom_include_headers);
2240+ const endpointUrl = isTextCompletion && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER ?
2084- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
2241+ `${apiUrl}/completions` :
2085- apiUrl = API_PERPLEXITY;
2242+ `${apiUrl}/chat/completions`;
2086- apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
2243+
2087- headers = {};
2244+ const controller = new AbortController();
2088- bodyParams = {
2245+ request.socket.removeAllListeners('close');
2089- reasoning_effort: request.body.reasoning_effort,
2246+ request.socket.on('close', function () {
2090- };
2247+ controller.abort();
2091- request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
2248+ });
2092- if (request.body.json_schema) {
2249+
2093- bodyParams['response_format'] = {
2250+ if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
2094- type: 'json_schema',
2251+ bodyParams['tools'] = request.body.tools;
2095- json_schema: {
2252+ bodyParams['tool_choice'] = request.body.tool_choice;
2096- schema: request.body.json_schema.value,
2097- },
2098- };
20992253 }
2100- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
2254+
2101- apiUrl = API_GROQ;
2255+ if (request.body.json_schema && !bodyParams['response_format']) {
2102- apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
2103- headers = {};
2104- bodyParams = {};
2105- if (request.body.json_schema) {
21062256 bodyParams['response_format'] = {
21072257 type: 'json_schema',
21082258 json_schema: {
21092259 name: request.body.json_schema.name,
2110- description: request.body.json_schema.description,
2111- schema: request.body.json_schema.value,
21122260 strict: request.body.json_schema.strict ?? true,
2113- },
2114- };
2115- }
2116- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
2117- apiUrl = API_FIREWORKS;
2118- apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
2119- headers = {};
2120- bodyParams = {};
2121- if (request.body.json_schema) {
2122- bodyParams['response_format'] = {
2123- type: 'json_schema',
2124- json_schema: {
2125- name: request.body.json_schema.name,
2126- description: request.body.json_schema.description,
21272261 schema: request.body.json_schema.value,
2128- strict: request.body.json_schema.strict ?? true,
21292262 },
21302263 };
21312264 }
2132- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
2133- apiUrl = API_NANOGPT;
2134- apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
2135- headers = {};
2136- bodyParams = {};
2137- if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
2138- request.body.model = `${request.body.model}:online`;
2139- }
2140- if (request.body.min_p !== undefined) {
2141- bodyParams['min_p'] = request.body.min_p;
2142- }
2143- if (request.body.top_a !== undefined) {
2144- bodyParams['top_a'] = request.body.top_a;
2145- }
2146- if (request.body.repetition_penalty !== undefined) {
2147- bodyParams['repetition_penalty'] = request.body.repetition_penalty;
2148- }
2149- const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2150- const isClaude3or4 = /claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2151- const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2152- if (enableSystemPromptCache && isClaude3or4) {
2153- bodyParams['cache_control'] = {
2154- 'enabled': true,
2155- 'ttl': cacheTTL,
2156- };
2157- }
2158- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
2159- apiUrl = API_POLLINATIONS;
2160- apiKey = 'NONE';
2161- headers = {
2162- 'Authorization': '',
2163- };
2164- bodyParams = {
2165- reasoning_effort: request.body.reasoning_effort,
2166- private: true,
2167- referrer: 'sillytavern',
2168- seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
2169- };
2170- if (request.body.json_schema) {
2171- setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2172- }
2173- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
2174- apiUrl = API_MOONSHOT;
2175- apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
2176- headers = {};
2177- bodyParams = {};
2178- request.body.json_schema
2179- ? setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema)
2180- : addAssistantPrefix(request.body.messages, [], 'partial');
2181- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
2182- apiUrl = API_COMETAPI;
2183- apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
2184- headers = {};
2185- bodyParams = {
2186- reasoning_effort: request.body.reasoning_effort,
2187- };
2188- throw new Error('This provider is temporarily disabled.');
2189- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZAI) {
2190- apiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING ? API_ZAI_CODING : API_ZAI_COMMON;
2191- apiKey = readSecret(request.user.directories, SECRET_KEYS.ZAI);
2192- headers = {
2193- 'Accept-Language': 'en-US,en',
2194- };
2195- bodyParams = {
2196- thinking: {
2197- type: request.body.include_reasoning ? 'enabled' : 'disabled',
2198- },
2199- };
2200- if (request.body.json_schema) {
2201- setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2202- }
2203- } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
2204- apiUrl = API_SILICONFLOW;
2205- apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
2206- headers = {};
2207- bodyParams = {};
2208- if (request.body.json_schema) {
2209- setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2210- }
2211- } else {
2212- console.warn('This chat completion source is not supported yet.');
2213- return response.status(400).send({ error: true });
2214- }
22152265
2216- // A few of OpenAIs reasoning models support reasoning effort
2266+ const requestBody = {
2217- if (request.body.reasoning_effort && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2267+ 'messages': isTextCompletion === false ? request.body.messages : undefined,
2218- if (OPENAI_REASONING_EFFORT_MODELS.includes(request.body.model)) {
2268+ 'prompt': isTextCompletion === true ? textPrompt : undefined,
2219- bodyParams['reasoning_effort'] = OPENAI_REASONING_EFFORT_MAP[request.body.reasoning_effort] ?? request.body.reasoning_effort;
2269+ 'model': request.body.model,
2220- }
2270+ 'temperature': request.body.temperature,
2221- }
2271+ 'max_tokens': request.body.max_tokens,
2272+ 'max_completion_tokens': request.body.max_completion_tokens,
2273+ 'stream': request.body.stream,
2274+ 'presence_penalty': request.body.presence_penalty,
2275+ 'frequency_penalty': request.body.frequency_penalty,
2276+ 'top_p': request.body.top_p,
2277+ 'top_k': request.body.top_k,
2278+ 'stop': isTextCompletion === false ? request.body.stop : undefined,
2279+ 'logit_bias': request.body.logit_bias,
2280+ 'seed': request.body.seed,
2281+ 'n': request.body.n,
2282+ ...bodyParams,
2283+ };
22222284
22232285 if (request.body.verbositychat_completion_source &&=== [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2224- if (OPENAI_VERBOSITY_MODELS.test(request.body.model)) {
2286+ excludeKeysByYaml(requestBody, request.body.custom_exclude_body);
2225- bodyParams['verbosity'] = request.body.verbosity;
22262287 }
2227- }
2228-
2229- if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
2230- console.warn('OpenAI API key is missing.');
2231- return response.status(400).send({ error: true });
2232- }
2233-
2234- // Add custom stop sequences
2235- if (Array.isArray(request.body.stop) && request.body.stop.length > 0) {
2236- bodyParams['stop'] = request.body.stop;
2237- }
2238-
2239- const textPrompt = isTextCompletion ? convertTextCompletionPrompt(request.body.messages) : '';
2240- const endpointUrl = isTextCompletion && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER ?
2241- `${apiUrl}/completions` :
2242- `${apiUrl}/chat/completions`;
2243-
2244- const controller = new AbortController();
2245- request.socket.removeAllListeners('close');
2246- request.socket.on('close', function () {
2247- controller.abort();
2248- });
2249-
2250- if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
2251- bodyParams['tools'] = request.body.tools;
2252- bodyParams['tool_choice'] = request.body.tool_choice;
2253- }
22542288
2255- if (request.body.json_schema && !bodyParams['response_format']) {
2289+ /** @type {import('node-fetch').RequestInit} */
22562290 bodyParams['response_format']const config = {
22572291 typemethod: 'json_schemapost',
22582292 json_schemaheaders: {
2259- name: request.body.json_schema.name,
2293+ 'Content-Type': 'application/json',
2260- strict: request.body.json_schema.strict ?? true,
2294+ 'Authorization': 'Bearer ' + apiKey,
22612295 schema: request.body.json_schema.valueheaders,
22622296 },
2297+ body: JSON.stringify(requestBody),
2298+ signal: controller.signal,
22632299 };
2264- }
2265-
2266- const requestBody = {
2267- 'messages': isTextCompletion === false ? request.body.messages : undefined,
2268- 'prompt': isTextCompletion === true ? textPrompt : undefined,
2269- 'model': request.body.model,
2270- 'temperature': request.body.temperature,
2271- 'max_tokens': request.body.max_tokens,
2272- 'max_completion_tokens': request.body.max_completion_tokens,
2273- 'stream': request.body.stream,
2274- 'presence_penalty': request.body.presence_penalty,
2275- 'frequency_penalty': request.body.frequency_penalty,
2276- 'top_p': request.body.top_p,
2277- 'top_k': request.body.top_k,
2278- 'stop': isTextCompletion === false ? request.body.stop : undefined,
2279- 'logit_bias': request.body.logit_bias,
2280- 'seed': request.body.seed,
2281- 'n': request.body.n,
2282- ...bodyParams,
2283- };
2284-
2285- if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2286- excludeKeysByYaml(requestBody, request.body.custom_exclude_body);
2287- }
2288-
2289- /** @type {import('node-fetch').RequestInit} */
2290- const config = {
2291- method: 'post',
2292- headers: {
2293- 'Content-Type': 'application/json',
2294- 'Authorization': 'Bearer ' + apiKey,
2295- ...headers,
2296- },
2297- body: JSON.stringify(requestBody),
2298- signal: controller.signal,
2299- };
23002300
23012301 console.debug('Chat Completion request:', requestBody);
23022302
2303- makeRequest(config, response, request);
2303+ const fetchResponse = await fetch(endpointUrl, config);
23042304
2305- /**
2305+ if (request.body.stream) {
2306- * Makes a fetch request to the OpenAI API endpoint.
2306+ console.info('Streaming request in progress');
2307- * @param {import('node-fetch').RequestInit} config Fetch config
2307+ return forwardFetchResponse(fetchResponse, response);
2308- * @param {express.Response} response Express response
2308+ }
2309- * @param {express.Request} request Express request
2310- */
2311- async function makeRequest(config, response, request) {
2312- try {
2313- controller.signal.throwIfAborted();
2314- const fetchResponse = await fetch(endpointUrl, config);
23152309
23162310 if (request.bodyfetchResponse.streamok) {
2317- console.info('Streaming request in progress');
2311+ /** @type {any} */
2318- forwardFetchResponse(fetchResponse, response);
2312+ const json = await fetchResponse.json();
2319- return;
2313+ console.debug('Chat Completion response:', json);
2320- }
2314+ return response.send(json);
2315+ } else {
2316+ const responseText = await fetchResponse.text();
2317+ const errorData = tryParse(responseText);
23212318
2322- if (fetchResponse.ok) {
2319+ const message = fetchResponse.statusText || 'Unknown error occurred';
2323- /** @type {any} */
2320+ const quota_error = fetchResponse.status === 429 && errorData?.error?.type === 'insufficient_quota';
2324- let json = await fetchResponse.json();
2321+ console.error('Chat completion request error: ', message, responseText);
2325- response.send(json);
2326- console.debug('Chat Completion response:', json);
2327- } else {
2328- await handleErrorResponse(fetchResponse);
2329- }
2330- } catch (error) {
2331- console.error('Generation failed', error);
2332- const message = error.code === 'ECONNREFUSED'
2333- ? `Connection refused: ${error.message}`
2334- : error.message || 'Unknown error occurred';
23352322
23362323 if (!response.headersSent) {
23372324 response.status(502).send({ error: { message }, ...errorquota_error: }quota_error });
2325+ } else if (!response.writableEnded) {
2326+ response.write(responseText);
23382327 } else {
23392328 response.end();
23402329 }
23412330 }
2342- }
2331+ } catch (error) {
2343-
2332+ console.error('Generation failed', error);
2344- /**
2333+ const message = error.code === 'ECONNREFUSED'
2345- * @param {import("node-fetch").Response} errorResponse
2334+ ? `Connection refused: ${error.message}`
2346- */
2335+ : error.message || 'Unknown error occurred';
2347- async function handleErrorResponse(errorResponse) {
2348- const responseText = await errorResponse.text();
2349- const errorData = tryParse(responseText);
2350-
2351- const message = errorResponse.statusText || 'Unknown error occurred';
2352- const quota_error = errorResponse.status === 429 && errorData?.error?.type === 'insufficient_quota';
2353- console.error('Chat completion request error: ', message, responseText);
23542336
23552337 if (!response.headersSent) {
23562338 response.status(502).send({ error: { message }, quota_error:...error quota_error} });
2357- } else if (!response.writableEnded) {
2358- response.write(responseText);
23592339 } else {
23602340 response.end();
23612341 }