Refactor CC API async route handlers (#4885) * Improve error handling in CC /status and /generate endpoints * Cancel pending status check on switching CC source

9046fe8d2de22b166e588839c8579c9b1da754c0

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 10a2f0f7406035ca064eecc2e85f52a707ae770b
3 files changed, +539 -557Ignore whitespace
public/script.js+1 -1
@@ -744,7 +744,7 @@ function initStandaloneMode() {
744 }744 }
745}745}
746746
747function cancelStatusCheck(reason = 'Manually cancelled status check') {747export function cancelStatusCheck(reason = 'Manually cancelled status check') {
748 abortStatusCheck?.abort(new AbortReason(reason));748 abortStatusCheck?.abort(new AbortReason(reason));
749 abortStatusCheck = new AbortController();749 abortStatusCheck = new AbortController();
750 setOnlineStatus('no_connection');750 setOnlineStatus('no_connection');
public/scripts/openai.js+2 -0
@@ -7,6 +7,7 @@ import { Fuse, DOMPurify } from '../lib.js';
77
8import {8import {
9 abortStatusCheck,9 abortStatusCheck,
10 cancelStatusCheck,
10 characters,11 characters,
11 event_types,12 event_types,
12 eventSource,13 eventSource,
@@ -6349,6 +6350,7 @@ export function initOpenAI() {
6349 });6350 });
63506351
6351 $('#chat_completion_source').on('change', function () {6352 $('#chat_completion_source').on('change', function () {
6353 cancelStatusCheck('Chat Completion source changed');
6352 model_list = [];6354 model_list = [];
6353 oai_settings.chat_completion_source = String($(this).find(':selected').val());6355 oai_settings.chat_completion_source = String($(this).find(':selected').val());
6354 toggleChatCompletionForms();6356 toggleChatCompletionForms();
src/endpoints/backends/chat-completions.js+536 -556
@@ -1560,210 +1560,210 @@ async function sendAzureOpenAIRequest(request, response) {
1560export const router = express.Router();1560export const router = express.Router();
15611561
1562router.post('/status', async function (request, statusResponse) {1562router.post('/status', async function (request, statusResponse) {
1563 if (!request.body) return statusResponse.sendStatus(400);1563 try {
15641564 if (!request.body) return statusResponse.sendStatus(400);
1565 let apiUrl = '';1565
1566 let apiKey = '';1566 let apiUrl = '';
1567 let headers = {};1567 let apiKey = '';
1568 let queryParams = {};1568 let headers = {};
15691569 let queryParams = {};
1570 if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {1570
1571 apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();1571 if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1572 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);1572 apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1573 headers = {};1573 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1574 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {1574 headers = {};
1575 apiUrl = 'https://openrouter.ai/api/v1';1575 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1576 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);1576 apiUrl = 'https://openrouter.ai/api/v1';
1577 // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests1577 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1578 headers = { ...OPENROUTER_HEADERS };1578 // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1579 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {1579 headers = { ...OPENROUTER_HEADERS };
1580 apiUrl = new URL(request.body.reverse_proxy || API_MISTRAL).toString();1580 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
1581 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);1581 apiUrl = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
1582 headers = {};1582 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);
1583 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {1583 headers = {};
1584 apiUrl = request.body.custom_url;1584 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
1585 apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);1585 apiUrl = request.body.custom_url;
1586 headers = {};1586 apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
1587 mergeObjectWithYaml(headers, request.body.custom_include_headers);1587 headers = {};
1588 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {1588 mergeObjectWithYaml(headers, request.body.custom_include_headers);
1589 apiUrl = API_COHERE_V1;1589 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {
1590 apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE);1590 apiUrl = API_COHERE_V1;
1591 headers = {};1591 apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE);
1592 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CHUTES) {1592 headers = {};
1593 apiUrl = API_CHUTES;1593 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CHUTES) {
1594 apiKey = readSecret(request.user.directories, SECRET_KEYS.CHUTES);1594 apiUrl = API_CHUTES;
1595 headers = {};1595 apiKey = readSecret(request.user.directories, SECRET_KEYS.CHUTES);
1596 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ELECTRONHUB) {1596 headers = {};
1597 apiUrl = API_ELECTRONHUB;1597 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ELECTRONHUB) {
1598 apiKey = readSecret(request.user.directories, SECRET_KEYS.ELECTRONHUB);1598 apiUrl = API_ELECTRONHUB;
1599 headers = {};1599 apiKey = readSecret(request.user.directories, SECRET_KEYS.ELECTRONHUB);
1600 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {1600 headers = {};
1601 apiUrl = API_NANOGPT;1601 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
1602 apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);1602 apiUrl = API_NANOGPT;
1603 headers = {};1603 apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
1604 queryParams = { detailed: true };1604 headers = {};
1605 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {1605 queryParams = { detailed: true };
1606 apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK.replace('/beta', '')).toString();1606 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {
1607 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);1607 apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK.replace('/beta', '')).toString();
1608 headers = {};1608 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
1609 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.XAI) {1609 headers = {};
1610 apiUrl = new URL(request.body.reverse_proxy || API_XAI).toString();1610 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.XAI) {
1611 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.XAI);1611 apiUrl = new URL(request.body.reverse_proxy || API_XAI).toString();
1612 headers = {};1612 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.XAI);
1613 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AIMLAPI) {1613 headers = {};
1614 apiUrl = API_AIMLAPI;1614 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AIMLAPI) {
1615 apiKey = readSecret(request.user.directories, SECRET_KEYS.AIMLAPI);1615 apiUrl = API_AIMLAPI;
1616 headers = { ...AIMLAPI_HEADERS };1616 apiKey = readSecret(request.user.directories, SECRET_KEYS.AIMLAPI);
1617 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {1617 headers = { ...AIMLAPI_HEADERS };
1618 apiUrl = 'https://text.pollinations.ai';1618 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
1619 apiKey = 'NONE';1619 apiUrl = 'https://text.pollinations.ai';
1620 headers = {};1620 apiKey = 'NONE';
1621 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {1621 headers = {};
1622 apiUrl = API_GROQ;1622 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
1623 apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);1623 apiUrl = API_GROQ;
1624 headers = {};1624 apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
1625 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {1625 headers = {};
1626 apiUrl = API_COMETAPI;1626 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
1627 apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);1627 apiUrl = API_COMETAPI;
1628 headers = {};1628 apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
1629 throw new Error('This provider is temporarily disabled.');1629 headers = {};
1630 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {1630 throw new Error('This provider is temporarily disabled.');
1631 apiUrl = API_MOONSHOT;1631 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
1632 apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);1632 apiUrl = API_MOONSHOT;
1633 headers = {};1633 apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
1634 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {1634 headers = {};
1635 apiUrl = API_FIREWORKS;1635 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
1636 apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);1636 apiUrl = API_FIREWORKS;
1637 headers = {};1637 apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
1638 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MAKERSUITE) {1638 headers = {};
1639 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);1639 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MAKERSUITE) {
1640 apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);1640 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
1641 const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');1641 apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE);
1642 const modelsUrl = !apiKey && request.body.reverse_proxy1642 const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta');
1643 ? `${apiUrl}/${apiVersion}/models`1643 const modelsUrl = !apiKey && request.body.reverse_proxy
1644 : `${apiUrl}/${apiVersion}/models?key=${apiKey}`;1644 ? `${apiUrl}/${apiVersion}/models`
16451645 : `${apiUrl}/${apiVersion}/models?key=${apiKey}`;
1646 if (!apiKey && !request.body.reverse_proxy) {1646
1647 console.warn('Google AI Studio API key is missing.');1647 if (!apiKey && !request.body.reverse_proxy) {
1648 return statusResponse.status(400).send({ error: true });1648 console.warn('Google AI Studio API key is missing.');
1649 }1649 return statusResponse.status(400).send({ error: true });
1650 }
16501651
1651 try {1652 try {
1652 const response = await fetch(modelsUrl);1653 const response = await fetch(modelsUrl);
16531654
1654 if (response.ok) {1655 if (response.ok) {
1655 /** @type {any} */1656 /** @type {any} */
1656 const data = await response.json();1657 const data = await response.json();
1657 // Transform Google AI Studio models to OpenAI format1658 // Transform Google AI Studio models to OpenAI format
1658 const models = data.models1659 const models = data.models
1659 ?.filter(model => model.supportedGenerationMethods?.includes('generateContent'))1660 ?.filter(model => model.supportedGenerationMethods?.includes('generateContent'))
1660 ?.map(model => ({1661 ?.map(model => ({
1661 id: model.name.replace('models/', ''),1662 id: model.name.replace('models/', ''),
1662 })) || [];1663 })) || [];
16631664
1664 console.info('Available Google AI Studio models:', models.map(m => m.id));1665 console.info('Available Google AI Studio models:', models.map(m => m.id));
1665 return statusResponse.send({ data: models });1666 return statusResponse.send({ data: models });
1666 } else {1667 } else {
1667 console.warn('Google AI Studio models endpoint failed:', response.status, response.statusText);1668 console.warn('Google AI Studio models endpoint failed:', response.status, response.statusText);
1669 return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
1670 }
1671 } catch (error) {
1672 console.error('Error fetching Google AI Studio models:', error);
1668 return statusResponse.send({ error: true, bypass: true, data: { data: [] } });1673 return statusResponse.send({ error: true, bypass: true, data: { data: [] } });
1669 }1674 }
1670 } catch (error) {1675 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AZURE_OPENAI) {
1671 console.error('Error fetching Google AI Studio models:', error);1676 const { azure_base_url, azure_deployment_name, azure_api_version } = request.body;
1672 return statusResponse.send({ error: true, bypass: true, data: { data: [] } });1677 const apiKey = readSecret(request.user.directories, SECRET_KEYS.AZURE_OPENAI);
1673 }1678
1674 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.AZURE_OPENAI) {1679 // 1) Validate configuration from the frontend
1675 const { azure_base_url, azure_deployment_name, azure_api_version } = request.body;1680 if (!apiKey || !azure_base_url || !azure_deployment_name || !azure_api_version) {
1676 const apiKey = readSecret(request.user.directories, SECRET_KEYS.AZURE_OPENAI);1681 console.warn('Azure OpenAI status check failed: missing config from frontend.');
16771682 return statusResponse.status(400).send({ error: true, message: 'Azure configuration is incomplete.' });
1678 // 1) Validate configuration from the frontend1683 }
1679 if (!apiKey || !azure_base_url || !azure_deployment_name || !azure_api_version) {1684 // 2) Build URLs using the URL API for consistency and robustness.
1680 console.warn('Azure OpenAI status check failed: missing config from frontend.');1685 const modelsUrl = new URL('/openai/models', azure_base_url);
1681 return statusResponse.status(400).send({ error: true, message: 'Azure configuration is incomplete.' });1686 modelsUrl.searchParams.set('api-version', azure_api_version);
1682 }1687
1683 // 2) Build URLs using the URL API for consistency and robustness.1688 const chatUrl = new URL(`/openai/deployments/${azure_deployment_name}/chat/completions`, azure_base_url);
1684 const modelsUrl = new URL('/openai/models', azure_base_url);1689 chatUrl.searchParams.set('api-version', azure_api_version);
1685 modelsUrl.searchParams.set('api-version', azure_api_version);1690
16861691 // Map common status codes to user-friendly error messages
1687 const chatUrl = new URL(`/openai/deployments/${azure_deployment_name}/chat/completions`, azure_base_url);1692 const azureStatusErrorMap = {
1688 chatUrl.searchParams.set('api-version', azure_api_version);1693 400: 'API version may be invalid for this resource.',
16891694 401: 'Invalid API key or insufficient permissions.',
1690 // Map common status codes to user-friendly error messages1695 403: 'Invalid API key or insufficient permissions.',
1691 const azureStatusErrorMap = {1696 404: 'Endpoint URL appears incorrect (404).',
1692 400: 'API version may be invalid for this resource.',1697 };
1693 401: 'Invalid API key or insufficient permissions.',
1694 403: 'Invalid API key or insufficient permissions.',
1695 404: 'Endpoint URL appears incorrect (404).',
1696 };
16971698
1698 try {1699 try {
1699 // ---- A) GET /models: fast sanity check for endpoint + api key + api version ----1700 // ---- A) GET /models: fast sanity check for endpoint + api key + api version ----
1700 const apiConfigTest = await fetch(modelsUrl, {1701 const apiConfigTest = await fetch(modelsUrl, {
1701 method: 'GET',1702 method: 'GET',
1702 headers: { 'api-key': apiKey, 'Accept': 'application/json' },1703 headers: { 'api-key': apiKey, 'Accept': 'application/json' },
1703 });1704 });
17041705
1705 if (!apiConfigTest.ok) {1706 if (!apiConfigTest.ok) {
1706 let errText = '';1707 let errText = '';
1707 try { errText = await apiConfigTest.text(); } catch { /* response body may be empty */ }1708 try { errText = await apiConfigTest.text(); } catch { /* response body may be empty */ }
17081709
1709 console.warn('Azure OpenAI GET /models failed:', apiConfigTest.status, apiConfigTest.statusText, errText || '');1710 console.warn('Azure OpenAI GET /models failed:', apiConfigTest.status, apiConfigTest.statusText, errText || '');
17101711
1711 const defaultMessage = `Azure Models endpoint error: ${apiConfigTest.statusText}`;1712 const defaultMessage = `Azure Models endpoint error: ${apiConfigTest.statusText}`;
1712 const message = azureStatusErrorMap[apiConfigTest.status] ?? defaultMessage;1713 const message = azureStatusErrorMap[apiConfigTest.status] ?? defaultMessage;
1713 return statusResponse.status(apiConfigTest.status).send({ error: true, message });1714 return statusResponse.status(apiConfigTest.status).send({ error: true, message });
1714 }1715 }
17151716
1716 // ---- B) POST /chat/completions: verify deployment + read underlying model ID ----1717 // ---- B) POST /chat/completions: verify deployment + read underlying model ID ----
1717 // Small, deterministic probe to minimize cost/latency1718 // Small, deterministic probe to minimize cost/latency
1718 const modelPayload = {1719 const modelPayload = {
1719 messages: [{ role: 'user', content: 'Say word Hi' }],1720 messages: [{ role: 'user', content: 'Say word Hi' }],
1720 stream: false,1721 stream: false,
1721 max_completion_tokens: 5,1722 max_completion_tokens: 5,
1722 };1723 };
17231724
1724 const modelRequest = await fetch(chatUrl, {1725 const modelRequest = await fetch(chatUrl, {
1725 method: 'POST',1726 method: 'POST',
1726 headers: { 'api-key': apiKey, 'Content-Type': 'application/json', 'Accept': 'application/json' },1727 headers: { 'api-key': apiKey, 'Content-Type': 'application/json', 'Accept': 'application/json' },
1727 body: JSON.stringify(modelPayload),1728 body: JSON.stringify(modelPayload),
1728 });1729 });
17291730
1730 let modelResponse;1731 let modelResponse;
1731 try {1732 try {
1732 modelResponse = await modelRequest.json();1733 modelResponse = await modelRequest.json();
1733 } catch {1734 } catch {
1734 modelResponse = { raw: 'Failed to parse JSON response from chat completions probe.' };1735 modelResponse = { raw: 'Failed to parse JSON response from chat completions probe.' };
1735 }1736 }
17361737
1737 const modelId = /** @type {any} */ (modelResponse)?.model;1738 const modelId = /** @type {any} */ (modelResponse)?.model;
1738 if (!modelId) {1739 if (!modelId) {
1739 console.warn('Azure status check succeeded but could not find a model ID in the response.');1740 console.warn('Azure status check succeeded but could not find a model ID in the response.');
1740 console.debug('Azure Response Body:', modelResponse);1741 console.debug('Azure Response Body:', modelResponse);
1741 // Keep a benign success to avoid UX disruption in the UI1742 // Keep a benign success to avoid UX disruption in the UI
1742 return statusResponse.send({ data: [] });1743 return statusResponse.send({ data: [] });
1743 }1744 }
17441745
1745 console.info(color.green('Azure OpenAI connection successful. Detected model:'), modelId);1746 console.info(color.green('Azure OpenAI connection successful. Detected model:'), modelId);
1746 // Consistent response format: always an array of { id }1747 // Consistent response format: always an array of { id }
1747 return statusResponse.send({ data: [{ id: modelId }] });1748 return statusResponse.send({ data: [{ id: modelId }] });
1748 } catch (error) {1749 } catch (error) {
1749 console.error('Azure OpenAI status check connection error:', error);1750 console.error('Azure OpenAI status check connection error:', error);
1750 return statusResponse.status(500).send({ error: true, message: 'Failed to connect to the Azure endpoint.' });1751 return statusResponse.status(500).send({ error: true, message: 'Failed to connect to the Azure endpoint.' });
1752 }
1753 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
1754 apiUrl = API_SILICONFLOW;
1755 apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
1756 headers = {};
1757 } else {
1758 console.warn('This chat completion source is not supported yet.');
1759 return statusResponse.status(400).send({ error: true });
1751 }1760 }
1752 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
1753 apiUrl = API_SILICONFLOW;
1754 apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
1755 headers = {};
1756 } else {
1757 console.warn('This chat completion source is not supported yet.');
1758 return statusResponse.status(400).send({ error: true });
1759 }
17601761
1761 if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {1762 if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
1762 console.warn('Chat Completion API key is missing.');1763 console.warn('Chat Completion API key is missing.');
1763 return statusResponse.status(400).send({ error: true });1764 return statusResponse.status(400).send({ error: true });
1764 }1765 }
17651766
1766 try {
1767 const modelsUrl = new URL(urlJoin(apiUrl, '/models'));1767 const modelsUrl = new URL(urlJoin(apiUrl, '/models'));
1768 Object.keys(queryParams).forEach(key => {1768 Object.keys(queryParams).forEach(key => {
1769 modelsUrl.searchParams.append(key, queryParams[key]);1769 modelsUrl.searchParams.append(key, queryParams[key]);
@@ -1933,429 +1933,409 @@ router.post('/bias', async function (request, response) {
1933 }1933 }
1934});1934});
19351935
1936router.post('/generate', async function (request, response) {
1937 try {
1938 if (!request.body) return response.status(400).send({ error: true });
1939
1940 const postProcessingType = request.body.custom_prompt_post_processing;
1941 if (Array.isArray(request.body.messages) && postProcessingType) {
1942 console.info('Applying custom prompt post-processing of type', postProcessingType);
1943 request.body.messages = postProcessPrompt(
1944 request.body.messages,
1945 postProcessingType,
1946 getPromptNames(request));
1947 }
1948
1949 if (request.body.json_schema?.value) {
1950 request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
1951 }
1952
1953 switch (request.body.chat_completion_source) {
1954 case CHAT_COMPLETION_SOURCES.CLAUDE: return await sendClaudeRequest(request, response);
1955 case CHAT_COMPLETION_SOURCES.AI21: return await sendAI21Request(request, response);
1956 case CHAT_COMPLETION_SOURCES.MAKERSUITE: return await sendMakerSuiteRequest(request, response);
1957 case CHAT_COMPLETION_SOURCES.VERTEXAI: return await sendMakerSuiteRequest(request, response);
1958 case CHAT_COMPLETION_SOURCES.MISTRALAI: return await sendMistralAIRequest(request, response);
1959 case CHAT_COMPLETION_SOURCES.COHERE: return await sendCohereRequest(request, response);
1960 case CHAT_COMPLETION_SOURCES.DEEPSEEK: return await sendDeepSeekRequest(request, response);
1961 case CHAT_COMPLETION_SOURCES.AIMLAPI: return await sendAimlapiRequest(request, response);
1962 case CHAT_COMPLETION_SOURCES.XAI: return await sendXaiRequest(request, response);
1963 case CHAT_COMPLETION_SOURCES.CHUTES: return await sendChutesRequest(request, response);
1964 case CHAT_COMPLETION_SOURCES.ELECTRONHUB: return await sendElectronHubRequest(request, response);
1965 case CHAT_COMPLETION_SOURCES.AZURE_OPENAI: return await sendAzureOpenAIRequest(request, response);
1966 }
1967
1968 let apiUrl;
1969 let apiKey;
1970 let headers;
1971 let bodyParams;
1972 const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model)) || typeof request.body.messages === 'string';
1973
1974 if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1975 apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1976 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1977 headers = {};
1978 bodyParams = {
1979 logprobs: request.body.logprobs,
1980 top_logprobs: undefined,
1981 };
19361982
1937router.post('/generate', function (request, response) {1983 // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
1938 if (!request.body) return response.status(400).send({ error: true });1984 if (!isTextCompletion && bodyParams.logprobs > 0) {
1985 bodyParams.top_logprobs = bodyParams.logprobs;
1986 bodyParams.logprobs = true;
1987 }
19391988
1940 const postProcessingType = request.body.custom_prompt_post_processing;1989 if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {
1941 if (Array.isArray(request.body.messages) && postProcessingType) {1990 bodyParams['user'] = uuidv4();
1942 console.info('Applying custom prompt post-processing of type', postProcessingType);1991 }
1943 request.body.messages = postProcessPrompt(1992 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1944 request.body.messages,1993 apiUrl = 'https://openrouter.ai/api/v1';
1945 postProcessingType,1994 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1946 getPromptNames(request));1995 // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1947 }1996 headers = { ...OPENROUTER_HEADERS };
1997 bodyParams = {
1998 'transforms': getOpenRouterTransforms(request),
1999 'plugins': getOpenRouterPlugins(request),
2000 'include_reasoning': Boolean(request.body.include_reasoning),
2001 };
19482002
1949 if (request.body.json_schema?.value) {2003 if (request.body.min_p !== undefined) {
1950 request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);2004 bodyParams['min_p'] = request.body.min_p;
1951 }2005 }
19522006
1953 switch (request.body.chat_completion_source) {2007 if (request.body.top_a !== undefined) {
1954 case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response);2008 bodyParams['top_a'] = request.body.top_a;
1955 case CHAT_COMPLETION_SOURCES.AI21: return sendAI21Request(request, response);2009 }
1956 case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response);
1957 case CHAT_COMPLETION_SOURCES.VERTEXAI: return sendMakerSuiteRequest(request, response);
1958 case CHAT_COMPLETION_SOURCES.MISTRALAI: return sendMistralAIRequest(request, response);
1959 case CHAT_COMPLETION_SOURCES.COHERE: return sendCohereRequest(request, response);
1960 case CHAT_COMPLETION_SOURCES.DEEPSEEK: return sendDeepSeekRequest(request, response);
1961 case CHAT_COMPLETION_SOURCES.AIMLAPI: return sendAimlapiRequest(request, response);
1962 case CHAT_COMPLETION_SOURCES.XAI: return sendXaiRequest(request, response);
1963 case CHAT_COMPLETION_SOURCES.CHUTES: return sendChutesRequest(request, response);
1964 case CHAT_COMPLETION_SOURCES.ELECTRONHUB: return sendElectronHubRequest(request, response);
1965 case CHAT_COMPLETION_SOURCES.AZURE_OPENAI: return sendAzureOpenAIRequest(request, response);
1966 }
19672010
1968 let apiUrl;2011 if (request.body.repetition_penalty !== undefined) {
1969 let apiKey;2012 bodyParams['repetition_penalty'] = request.body.repetition_penalty;
1970 let headers;2013 }
1971 let bodyParams;
1972 const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model)) || typeof request.body.messages === 'string';
1973
1974 if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
1975 apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
1976 apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
1977 headers = {};
1978 bodyParams = {
1979 logprobs: request.body.logprobs,
1980 top_logprobs: undefined,
1981 };
19822014
1983 // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }2015 if (Array.isArray(request.body.provider) && request.body.provider.length > 0) {
1984 if (!isTextCompletion && bodyParams.logprobs > 0) {2016 bodyParams['provider'] = {
1985 bodyParams.top_logprobs = bodyParams.logprobs;2017 allow_fallbacks: request.body.allow_fallbacks ?? true,
1986 bodyParams.logprobs = true;2018 order: request.body.provider ?? [],
1987 }2019 };
2020 }
19882021
1989 if (getConfigValue('openai.randomizeUserId', false, 'boolean')) {2022 if (request.body.use_fallback) {
1990 bodyParams['user'] = uuidv4();2023 bodyParams['route'] = 'fallback';
1991 }2024 }
1992 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
1993 apiUrl = 'https://openrouter.ai/api/v1';
1994 apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
1995 // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
1996 headers = { ...OPENROUTER_HEADERS };
1997 bodyParams = {
1998 'transforms': getOpenRouterTransforms(request),
1999 'plugins': getOpenRouterPlugins(request),
2000 'include_reasoning': Boolean(request.body.include_reasoning),
2001 };
20022025
2003 if (request.body.min_p !== undefined) {2026 if (request.body.reasoning_effort) {
2004 bodyParams['min_p'] = request.body.min_p;2027 bodyParams['reasoning'] = { effort: request.body.reasoning_effort };
2005 }2028 }
20062029
2007 if (request.body.top_a !== undefined) {2030 if (request.body.verbosity) {
2008 bodyParams['top_a'] = request.body.top_a;2031 bodyParams['verbosity'] = request.body.verbosity;
2009 }2032 }
20102033
2011 if (request.body.repetition_penalty !== undefined) {2034 if (request.body.json_schema) {
2012 bodyParams['repetition_penalty'] = request.body.repetition_penalty;2035 bodyParams['response_format'] = {
2013 }2036 type: 'json_schema',
2037 json_schema: {
2038 name: request.body.json_schema.name,
2039 strict: request.body.json_schema.strict ?? true,
2040 schema: request.body.json_schema.value,
2041 },
2042 };
2043 }
20142044
2015 if (Array.isArray(request.body.provider) && request.body.provider.length > 0) {2045 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2016 bodyParams['provider'] = {2046 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
2017 allow_fallbacks: request.body.allow_fallbacks ?? true,2047 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2018 order: request.body.provider ?? [],2048 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2019 };2049 if (Array.isArray(request.body.messages)) {
2020 }2050 embedOpenRouterMedia(request.body.messages);
20212051
2022 if (request.body.use_fallback) {2052 if (isClaude3or4) {
2023 bodyParams['route'] = 'fallback';2053 if (enableSystemPromptCache) {
2024 }2054 cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
2055 }
20252056
2026 if (request.body.reasoning_effort) {2057 if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {
2027 bodyParams['reasoning'] = { effort: request.body.reasoning_effort };2058 cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);
2028 }2059 }
2060 }
2061 }
20292062
2030 if (request.body.verbosity) {2063 const isGemini = /google\/gemini/.test(request.body.model);
2031 bodyParams['verbosity'] = request.body.verbosity;2064 if (isGemini) {
2032 }2065 bodyParams['safety_settings'] = GEMINI_SAFETY;
2066 }
2067 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2068 apiUrl = request.body.custom_url;
2069 apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
2070 headers = {};
2071 bodyParams = {
2072 logprobs: request.body.logprobs,
2073 top_logprobs: undefined,
2074 };
20332075
2034 if (request.body.json_schema) {2076 // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
2035 bodyParams['response_format'] = {2077 if (!isTextCompletion && bodyParams.logprobs > 0) {
2036 type: 'json_schema',2078 bodyParams.top_logprobs = bodyParams.logprobs;
2037 json_schema: {2079 bodyParams.logprobs = true;
2038 name: request.body.json_schema.name,2080 }
2039 strict: request.body.json_schema.strict ?? true,2081
2040 schema: request.body.json_schema.value,2082 mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
2083 mergeObjectWithYaml(headers, request.body.custom_include_headers);
2084 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
2085 apiUrl = API_PERPLEXITY;
2086 apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
2087 headers = {};
2088 bodyParams = {
2089 reasoning_effort: request.body.reasoning_effort,
2090 };
2091 request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
2092 if (request.body.json_schema) {
2093 bodyParams['response_format'] = {
2094 type: 'json_schema',
2095 json_schema: {
2096 schema: request.body.json_schema.value,
2097 },
2098 };
2099 }
2100 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
2101 apiUrl = API_GROQ;
2102 apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
2103 headers = {};
2104 bodyParams = {};
2105 if (request.body.json_schema) {
2106 bodyParams['response_format'] = {
2107 type: 'json_schema',
2108 json_schema: {
2109 name: request.body.json_schema.name,
2110 description: request.body.json_schema.description,
2111 schema: request.body.json_schema.value,
2112 strict: request.body.json_schema.strict ?? true,
2113 },
2114 };
2115 }
2116 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
2117 apiUrl = API_FIREWORKS;
2118 apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
2119 headers = {};
2120 bodyParams = {};
2121 if (request.body.json_schema) {
2122 bodyParams['response_format'] = {
2123 type: 'json_schema',
2124 json_schema: {
2125 name: request.body.json_schema.name,
2126 description: request.body.json_schema.description,
2127 schema: request.body.json_schema.value,
2128 strict: request.body.json_schema.strict ?? true,
2129 },
2130 };
2131 }
2132 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
2133 apiUrl = API_NANOGPT;
2134 apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
2135 headers = {};
2136 bodyParams = {};
2137 if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
2138 request.body.model = `${request.body.model}:online`;
2139 }
2140 if (request.body.min_p !== undefined) {
2141 bodyParams['min_p'] = request.body.min_p;
2142 }
2143 if (request.body.top_a !== undefined) {
2144 bodyParams['top_a'] = request.body.top_a;
2145 }
2146 if (request.body.repetition_penalty !== undefined) {
2147 bodyParams['repetition_penalty'] = request.body.repetition_penalty;
2148 }
2149 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2150 const isClaude3or4 = /claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2151 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2152 if (enableSystemPromptCache && isClaude3or4) {
2153 bodyParams['cache_control'] = {
2154 'enabled': true,
2155 'ttl': cacheTTL,
2156 };
2157 }
2158 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
2159 apiUrl = API_POLLINATIONS;
2160 apiKey = 'NONE';
2161 headers = {
2162 'Authorization': '',
2163 };
2164 bodyParams = {
2165 reasoning_effort: request.body.reasoning_effort,
2166 private: true,
2167 referrer: 'sillytavern',
2168 seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
2169 };
2170 if (request.body.json_schema) {
2171 setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2172 }
2173 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
2174 apiUrl = API_MOONSHOT;
2175 apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
2176 headers = {};
2177 bodyParams = {};
2178 request.body.json_schema
2179 ? setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema)
2180 : addAssistantPrefix(request.body.messages, [], 'partial');
2181 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
2182 apiUrl = API_COMETAPI;
2183 apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
2184 headers = {};
2185 bodyParams = {
2186 reasoning_effort: request.body.reasoning_effort,
2187 };
2188 throw new Error('This provider is temporarily disabled.');
2189 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZAI) {
2190 apiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING ? API_ZAI_CODING : API_ZAI_COMMON;
2191 apiKey = readSecret(request.user.directories, SECRET_KEYS.ZAI);
2192 headers = {
2193 'Accept-Language': 'en-US,en',
2194 };
2195 bodyParams = {
2196 thinking: {
2197 type: request.body.include_reasoning ? 'enabled' : 'disabled',
2041 },2198 },
2042 };2199 };
2200 if (request.body.json_schema) {
2201 setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2202 }
2203 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
2204 apiUrl = API_SILICONFLOW;
2205 apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
2206 headers = {};
2207 bodyParams = {};
2208 if (request.body.json_schema) {
2209 setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2210 }
2211 } else {
2212 console.warn('This chat completion source is not supported yet.');
2213 return response.status(400).send({ error: true });
2043 }2214 }
20442215
2045 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');2216 // A few of OpenAIs reasoning models support reasoning effort
2046 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');2217 if (request.body.reasoning_effort && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2047 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);2218 if (OPENAI_REASONING_EFFORT_MODELS.includes(request.body.model)) {
2048 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';2219 bodyParams['reasoning_effort'] = OPENAI_REASONING_EFFORT_MAP[request.body.reasoning_effort] ?? request.body.reasoning_effort;
2049 if (Array.isArray(request.body.messages)) {2220 }
2050 embedOpenRouterMedia(request.body.messages);2221 }
2051
2052 if (isClaude3or4) {
2053 if (enableSystemPromptCache) {
2054 cachingSystemPromptForOpenRouterClaude(request.body.messages, cacheTTL);
2055 }
20562222
2057 if (Number.isInteger(cachingAtDepth) && cachingAtDepth >= 0) {2223 if (request.body.verbosity && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {
2058 cachingAtDepthForOpenRouterClaude(request.body.messages, cachingAtDepth, cacheTTL);2224 if (OPENAI_VERBOSITY_MODELS.test(request.body.model)) {
2059 }2225 bodyParams['verbosity'] = request.body.verbosity;
2060 }2226 }
2061 }2227 }
20622228
2063 const isGemini = /google\/gemini/.test(request.body.model);2229 if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
2064 if (isGemini) {2230 console.warn('OpenAI API key is missing.');
2065 bodyParams['safety_settings'] = GEMINI_SAFETY;2231 return response.status(400).send({ error: true });
2066 }2232 }
2067 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2068 apiUrl = request.body.custom_url;
2069 apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
2070 headers = {};
2071 bodyParams = {
2072 logprobs: request.body.logprobs,
2073 top_logprobs: undefined,
2074 };
20752233
2076 // Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }2234 // Add custom stop sequences
2077 if (!isTextCompletion && bodyParams.logprobs > 0) {2235 if (Array.isArray(request.body.stop) && request.body.stop.length > 0) {
2078 bodyParams.top_logprobs = bodyParams.logprobs;2236 bodyParams['stop'] = request.body.stop;
2079 bodyParams.logprobs = true;
2080 }2237 }
20812238
2082 mergeObjectWithYaml(bodyParams, request.body.custom_include_body);2239 const textPrompt = isTextCompletion ? convertTextCompletionPrompt(request.body.messages) : '';
2083 mergeObjectWithYaml(headers, request.body.custom_include_headers);2240 const endpointUrl = isTextCompletion && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER ?
2084 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {2241 `${apiUrl}/completions` :
2085 apiUrl = API_PERPLEXITY;2242 `${apiUrl}/chat/completions`;
2086 apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);2243
2087 headers = {};2244 const controller = new AbortController();
2088 bodyParams = {2245 request.socket.removeAllListeners('close');
2089 reasoning_effort: request.body.reasoning_effort,2246 request.socket.on('close', function () {
2090 };2247 controller.abort();
2091 request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));2248 });
2092 if (request.body.json_schema) {2249
2093 bodyParams['response_format'] = {2250 if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
2094 type: 'json_schema',2251 bodyParams['tools'] = request.body.tools;
2095 json_schema: {2252 bodyParams['tool_choice'] = request.body.tool_choice;
2096 schema: request.body.json_schema.value,
2097 },
2098 };
2099 }2253 }
2100 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {2254
2101 apiUrl = API_GROQ;2255 if (request.body.json_schema && !bodyParams['response_format']) {
2102 apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
2103 headers = {};
2104 bodyParams = {};
2105 if (request.body.json_schema) {
2106 bodyParams['response_format'] = {2256 bodyParams['response_format'] = {
2107 type: 'json_schema',2257 type: 'json_schema',
2108 json_schema: {2258 json_schema: {
2109 name: request.body.json_schema.name,2259 name: request.body.json_schema.name,
2110 description: request.body.json_schema.description,
2111 schema: request.body.json_schema.value,
2112 strict: request.body.json_schema.strict ?? true,2260 strict: request.body.json_schema.strict ?? true,
2113 },
2114 };
2115 }
2116 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.FIREWORKS) {
2117 apiUrl = API_FIREWORKS;
2118 apiKey = readSecret(request.user.directories, SECRET_KEYS.FIREWORKS);
2119 headers = {};
2120 bodyParams = {};
2121 if (request.body.json_schema) {
2122 bodyParams['response_format'] = {
2123 type: 'json_schema',
2124 json_schema: {
2125 name: request.body.json_schema.name,
2126 description: request.body.json_schema.description,
2127 schema: request.body.json_schema.value,2261 schema: request.body.json_schema.value,
2128 strict: request.body.json_schema.strict ?? true,
2129 },2262 },
2130 };2263 };
2131 }2264 }
2132 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
2133 apiUrl = API_NANOGPT;
2134 apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
2135 headers = {};
2136 bodyParams = {};
2137 if (request.body.enable_web_search && !/:online$/.test(request.body.model)) {
2138 request.body.model = `${request.body.model}:online`;
2139 }
2140 if (request.body.min_p !== undefined) {
2141 bodyParams['min_p'] = request.body.min_p;
2142 }
2143 if (request.body.top_a !== undefined) {
2144 bodyParams['top_a'] = request.body.top_a;
2145 }
2146 if (request.body.repetition_penalty !== undefined) {
2147 bodyParams['repetition_penalty'] = request.body.repetition_penalty;
2148 }
2149 const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false, 'boolean');
2150 const isClaude3or4 = /claude-(3|opus-4|sonnet-4|haiku-4)/.test(request.body.model);
2151 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
2152 if (enableSystemPromptCache && isClaude3or4) {
2153 bodyParams['cache_control'] = {
2154 'enabled': true,
2155 'ttl': cacheTTL,
2156 };
2157 }
2158 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.POLLINATIONS) {
2159 apiUrl = API_POLLINATIONS;
2160 apiKey = 'NONE';
2161 headers = {
2162 'Authorization': '',
2163 };
2164 bodyParams = {
2165 reasoning_effort: request.body.reasoning_effort,
2166 private: true,
2167 referrer: 'sillytavern',
2168 seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
2169 };
2170 if (request.body.json_schema) {
2171 setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2172 }
2173 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MOONSHOT) {
2174 apiUrl = API_MOONSHOT;
2175 apiKey = readSecret(request.user.directories, SECRET_KEYS.MOONSHOT);
2176 headers = {};
2177 bodyParams = {};
2178 request.body.json_schema
2179 ? setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema)
2180 : addAssistantPrefix(request.body.messages, [], 'partial');
2181 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COMETAPI) {
2182 apiUrl = API_COMETAPI;
2183 apiKey = readSecret(request.user.directories, SECRET_KEYS.COMETAPI);
2184 headers = {};
2185 bodyParams = {
2186 reasoning_effort: request.body.reasoning_effort,
2187 };
2188 throw new Error('This provider is temporarily disabled.');
2189 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZAI) {
2190 apiUrl = request.body.zai_endpoint === ZAI_ENDPOINT.CODING ? API_ZAI_CODING : API_ZAI_COMMON;
2191 apiKey = readSecret(request.user.directories, SECRET_KEYS.ZAI);
2192 headers = {
2193 'Accept-Language': 'en-US,en',
2194 };
2195 bodyParams = {
2196 thinking: {
2197 type: request.body.include_reasoning ? 'enabled' : 'disabled',
2198 },
2199 };
2200 if (request.body.json_schema) {
2201 setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2202 }
2203 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.SILICONFLOW) {
2204 apiUrl = API_SILICONFLOW;
2205 apiKey = readSecret(request.user.directories, SECRET_KEYS.SILICONFLOW);
2206 headers = {};
2207 bodyParams = {};
2208 if (request.body.json_schema) {
2209 setJsonObjectFormat(bodyParams, request.body.messages, request.body.json_schema);
2210 }
2211 } else {
2212 console.warn('This chat completion source is not supported yet.');
2213 return response.status(400).send({ error: true });
2214 }
22152265
2216 // A few of OpenAIs reasoning models support reasoning effort2266 const requestBody = {
2217 if (request.body.reasoning_effort && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {2267 'messages': isTextCompletion === false ? request.body.messages : undefined,
2218 if (OPENAI_REASONING_EFFORT_MODELS.includes(request.body.model)) {2268 'prompt': isTextCompletion === true ? textPrompt : undefined,
2219 bodyParams['reasoning_effort'] = OPENAI_REASONING_EFFORT_MAP[request.body.reasoning_effort] ?? request.body.reasoning_effort;2269 'model': request.body.model,
2220 }2270 'temperature': request.body.temperature,
2221 }2271 'max_tokens': request.body.max_tokens,
2272 'max_completion_tokens': request.body.max_completion_tokens,
2273 'stream': request.body.stream,
2274 'presence_penalty': request.body.presence_penalty,
2275 'frequency_penalty': request.body.frequency_penalty,
2276 'top_p': request.body.top_p,
2277 'top_k': request.body.top_k,
2278 'stop': isTextCompletion === false ? request.body.stop : undefined,
2279 'logit_bias': request.body.logit_bias,
2280 'seed': request.body.seed,
2281 'n': request.body.n,
2282 ...bodyParams,
2283 };
22222284
2223 if (request.body.verbosity && [CHAT_COMPLETION_SOURCES.CUSTOM, CHAT_COMPLETION_SOURCES.OPENAI].includes(request.body.chat_completion_source)) {2285 if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2224 if (OPENAI_VERBOSITY_MODELS.test(request.body.model)) {2286 excludeKeysByYaml(requestBody, request.body.custom_exclude_body);
2225 bodyParams['verbosity'] = request.body.verbosity;
2226 }2287 }
2227 }
2228
2229 if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
2230 console.warn('OpenAI API key is missing.');
2231 return response.status(400).send({ error: true });
2232 }
2233
2234 // Add custom stop sequences
2235 if (Array.isArray(request.body.stop) && request.body.stop.length > 0) {
2236 bodyParams['stop'] = request.body.stop;
2237 }
2238
2239 const textPrompt = isTextCompletion ? convertTextCompletionPrompt(request.body.messages) : '';
2240 const endpointUrl = isTextCompletion && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER ?
2241 `${apiUrl}/completions` :
2242 `${apiUrl}/chat/completions`;
2243
2244 const controller = new AbortController();
2245 request.socket.removeAllListeners('close');
2246 request.socket.on('close', function () {
2247 controller.abort();
2248 });
2249
2250 if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
2251 bodyParams['tools'] = request.body.tools;
2252 bodyParams['tool_choice'] = request.body.tool_choice;
2253 }
22542288
2255 if (request.body.json_schema && !bodyParams['response_format']) {2289 /** @type {import('node-fetch').RequestInit} */
2256 bodyParams['response_format'] = {2290 const config = {
2257 type: 'json_schema',2291 method: 'post',
2258 json_schema: {2292 headers: {
2259 name: request.body.json_schema.name,2293 'Content-Type': 'application/json',
2260 strict: request.body.json_schema.strict ?? true,2294 'Authorization': 'Bearer ' + apiKey,
2261 schema: request.body.json_schema.value,2295 ...headers,
2262 },2296 },
2297 body: JSON.stringify(requestBody),
2298 signal: controller.signal,
2263 };2299 };
2264 }
2265
2266 const requestBody = {
2267 'messages': isTextCompletion === false ? request.body.messages : undefined,
2268 'prompt': isTextCompletion === true ? textPrompt : undefined,
2269 'model': request.body.model,
2270 'temperature': request.body.temperature,
2271 'max_tokens': request.body.max_tokens,
2272 'max_completion_tokens': request.body.max_completion_tokens,
2273 'stream': request.body.stream,
2274 'presence_penalty': request.body.presence_penalty,
2275 'frequency_penalty': request.body.frequency_penalty,
2276 'top_p': request.body.top_p,
2277 'top_k': request.body.top_k,
2278 'stop': isTextCompletion === false ? request.body.stop : undefined,
2279 'logit_bias': request.body.logit_bias,
2280 'seed': request.body.seed,
2281 'n': request.body.n,
2282 ...bodyParams,
2283 };
2284
2285 if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
2286 excludeKeysByYaml(requestBody, request.body.custom_exclude_body);
2287 }
2288
2289 /** @type {import('node-fetch').RequestInit} */
2290 const config = {
2291 method: 'post',
2292 headers: {
2293 'Content-Type': 'application/json',
2294 'Authorization': 'Bearer ' + apiKey,
2295 ...headers,
2296 },
2297 body: JSON.stringify(requestBody),
2298 signal: controller.signal,
2299 };
23002300
2301 console.debug('Chat Completion request:', requestBody);2301 console.debug('Chat Completion request:', requestBody);
23022302
2303 makeRequest(config, response, request);2303 const fetchResponse = await fetch(endpointUrl, config);
23042304
2305 /**2305 if (request.body.stream) {
2306 * Makes a fetch request to the OpenAI API endpoint.2306 console.info('Streaming request in progress');
2307 * @param {import('node-fetch').RequestInit} config Fetch config2307 return forwardFetchResponse(fetchResponse, response);
2308 * @param {express.Response} response Express response2308 }
2309 * @param {express.Request} request Express request
2310 */
2311 async function makeRequest(config, response, request) {
2312 try {
2313 controller.signal.throwIfAborted();
2314 const fetchResponse = await fetch(endpointUrl, config);
23152309
2316 if (request.body.stream) {2310 if (fetchResponse.ok) {
2317 console.info('Streaming request in progress');2311 /** @type {any} */
2318 forwardFetchResponse(fetchResponse, response);2312 const json = await fetchResponse.json();
2319 return;2313 console.debug('Chat Completion response:', json);
2320 }2314 return response.send(json);
2315 } else {
2316 const responseText = await fetchResponse.text();
2317 const errorData = tryParse(responseText);
23212318
2322 if (fetchResponse.ok) {2319 const message = fetchResponse.statusText || 'Unknown error occurred';
2323 /** @type {any} */2320 const quota_error = fetchResponse.status === 429 && errorData?.error?.type === 'insufficient_quota';
2324 let json = await fetchResponse.json();2321 console.error('Chat completion request error: ', message, responseText);
2325 response.send(json);
2326 console.debug('Chat Completion response:', json);
2327 } else {
2328 await handleErrorResponse(fetchResponse);
2329 }
2330 } catch (error) {
2331 console.error('Generation failed', error);
2332 const message = error.code === 'ECONNREFUSED'
2333 ? `Connection refused: ${error.message}`
2334 : error.message || 'Unknown error occurred';
23352322
2336 if (!response.headersSent) {2323 if (!response.headersSent) {
2337 response.status(502).send({ error: { message, ...error } });2324 response.send({ error: { message }, quota_error: quota_error });
2325 } else if (!response.writableEnded) {
2326 response.write(responseText);
2338 } else {2327 } else {
2339 response.end();2328 response.end();
2340 }2329 }
2341 }2330 }
2342 }2331 } catch (error) {
23432332 console.error('Generation failed', error);
2344 /**2333 const message = error.code === 'ECONNREFUSED'
2345 * @param {import("node-fetch").Response} errorResponse2334 ? `Connection refused: ${error.message}`
2346 */2335 : error.message || 'Unknown error occurred';
2347 async function handleErrorResponse(errorResponse) {
2348 const responseText = await errorResponse.text();
2349 const errorData = tryParse(responseText);
2350
2351 const message = errorResponse.statusText || 'Unknown error occurred';
2352 const quota_error = errorResponse.status === 429 && errorData?.error?.type === 'insufficient_quota';
2353 console.error('Chat completion request error: ', message, responseText);
23542336
2355 if (!response.headersSent) {2337 if (!response.headersSent) {
2356 response.send({ error: { message }, quota_error: quota_error });2338 response.status(502).send({ error: { message, ...error } });
2357 } else if (!response.writableEnded) {
2358 response.write(responseText);
2359 } else {2339 } else {
2360 response.end();2340 response.end();
2361 }2341 }