Added structured output for common APIs (#4272) * Added structured output for common APIs * eslint * Added frontend impl * Type name change * Unprefix json_schema, apply review suggestions * Add schema to generateQuietPrompt, add comments * Prettify diff * Extract JSON from Claude response * Add structured gen for Mistral * Hack to support schema for DeepSeek * Hack JSON schema for AI21 * Add Groq structured gen * Add JSON mode for pollinations * Add JSON schema for perplexity * Add JSON schema for AIML * Using extractJsonFromData in custom-request, added google rules for flattenSchema * Fix response parsing * Fix Google * Fixed json parse * Expose generateRaw to getContext --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

cd176039ef0fef85c74bb3315abc535aca912fc8

bmen25124 <bmen25124@gmail.com>

Signed
Parents: 87f8a3eb2c30d83bd52a7ebbbfeab3131162364b
6 files changed, +329 -26Ignore whitespace
public/script.js+111 -19
@@ -2330,29 +2330,31 @@ export function getStoppingStrings(isImpersonate, isContinue) {
23302330
23312331/**
23322332 * Background generation based on the provided prompt.
23332333 * @param {string} quiet_promptquietPrompt Instruction prompt for the AI
23342334 * @param {boolean} [quietToLoud] Whether the message should be sent in a foreground (loud) or background (quiet) mode
23352335 * @param {boolean} [skipWIAN] whetherWhether to skip addition of World Info and Author's Note into the prompt
23362336 * @param {string} [quietImage] Image to use for the quiet prompt
23372337 * @param {string} [quietName] Name to use for the quiet prompt (defaults to "System:")
23382338 * @param {number} [responseLength] Maximum response length. If unset, the global default value is used.
23392339 * @param {number} force_chid[forceChId] Character ID to use for this generation run. Works in groups only.
2340- * @returns
2340+ * @param {AdditionalRequestOptions} [options={}] Additional generation request options.
2341+ * @returns {Promise<string>} Generated text. If using structured output, will contain a serialized JSON object.
23412342 */
23422343export async function generateQuietPrompt(quiet_promptquietPrompt, quietToLoud = false, skipWIAN = false, quietImage = null, quietName = null, responseLength = null, force_chidforceChId = null, { jsonSchema } = {}) {
23432344 console.log('got into genQuietPrompt');
23442345 const responseLengthCustomized = typeof responseLength === 'number' && responseLength > 0;
23452346 let eventHook = () => { };
23462347 try {
23472348 /** @type {GenerateOptions} */
23482349 const options = {
23492350 quiet_prompt: quietPrompt,
23502351 quietToLoud,
23512352 skipWIAN: skipWIAN,
23522353 force_name2: true,
23532354 quietImage: quietImage,
23542355 quietName: quietName,
23552356 force_chid: force_chidforceChId,
2357+ jsonSchema: jsonSchema,
23562358 };
23572359 if (responseLengthCustomized) {
23582360 TempResponseLength.save(main_api, responseLength);
@@ -3127,9 +3129,10 @@ export function createRawPrompt(prompt, api, instructOverride, quietToLoud, syst
31273129 * @param {number} [responseLength] Maximum response length. If unset, the global default value is used.
31283130 * @param {boolean} [trimNames] Whether to allow trimming "{{user}}:" and "{{char}}:" from the response.
31293131 * @param {string} [prefill] An optional prefill for the prompt.
3132+ * @param {AdditionalRequestOptions} [options] Additional options for generation
31303133 * @returns {Promise<string>} Generated message
31313134 */
31323135export async function generateRaw(prompt, api, instructOverride, quietToLoud, systemPrompt, responseLength, trimNames = true, prefill = '', options = {}) {
31333136 if (!api) {
31343137 api = main_api;
31353138 }
@@ -3181,7 +3184,7 @@ export async function generateRaw(prompt, api, instructOverride, quietToLoud, sy
31813184 if (api === 'koboldhorde') {
31823185 data = await generateHorde(prompt.toString(), generateData, abortController.signal, false);
31833186 } else if (api === 'openai') {
31843187 data = await sendOpenAIRequest('quiet', generateData, abortController.signal, options);
31853188 } else {
31863189 const generateUrl = getGenerateUrl(api);
31873190 const response = await fetch(generateUrl, {
@@ -3206,6 +3209,10 @@ export async function generateRaw(prompt, api, instructOverride, quietToLoud, sy
32063209 throw new Error(data.response);
32073210 }
32083211
3212+ if (options?.jsonSchema) {
3213+ return extractJsonFromData(data, { mainApi: api });
3214+ }
3215+
32093216 // format result, exclude user prompt bias
32103217 const message = cleanUpMessage({
32113218 getMessage: extractMessageFromData(data),
@@ -3338,15 +3345,35 @@ function removeLastMessage() {
33383345}
33393346
33403347/**
3348+ * @typedef {object} JsonSchema
3349+ * @property {string} name Name of the schema.
3350+ * @property {object} value JSON schema value.
3351+ * @property {string} [description] Description of the schema.
3352+ * @property {boolean} [strict] If true, the schema will be used in strict mode, meaning that only the fields defined in the schema will be allowed.
3353+ *
3354+ * @typedef {object} GenerateOptions
3355+ * @property {boolean} [automatic_trigger] If the generation was triggered automatically (e.g. group auto mode).
3356+ * @property {boolean} [force_name2] If a char name should be forced to add to the prompt's last line (Text Completion, non-Instruct only).
3357+ * @property {string} [quiet_prompt] A system instruction to use for the quiet prompt.
3358+ * @property {boolean} [quietToLoud] Whether the system instruction should be sent in background (quiet) or a foreground (loud) mode.
3359+ * @property {boolean} [skipWIAN] Skip adding World Info and Author's Note to the prompt.
3360+ * @property {number} [force_chid] Force character ID to use for the generation. Only works in groups.
3361+ * @property {AbortSignal} [signal] Abort signal to cancel the generation. If not provided, will create a new AbortController.
3362+ * @property {string} [quietImage] Image URL to use for the quiet prompt (defaults to empty string)
3363+ * @property {string} [quietName] Name to use for the quiet prompt (defaults to "System:")
3364+ * @property {number} [depth] Recursion depth for the generation. Used to prevent infinite loops in tool calls.
3365+ * @property {JsonSchema} [jsonSchema] JSON schema to use for the structured generation. Usually requires a special instruction.
3366+ */
3367+
3368+/**
33413369 * MARK:Generate()
33423370 * Runs a generation using the current chat context.
33433371 * @param {string} type Generation type
33443372 * @param {GenerateOptions} options Generation options
33453373 * @param {boolean} dryRun Whether to actually generate a message or just assemble the prompt
33463374 * @returns {Promise<any>} Returns a promise that resolves when the text is done generating.
3347- * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, quietName?: string, depth?: number }} GenerateOptions
33483375 */
33493376export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, jsonSchema = null, depth = 0 } = {}, dryRun = false) {
33503377 console.log('Generate entered');
33513378 setGenerationProgress(0);
33523379 generation_started = new Date();
@@ -4488,7 +4515,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
44884515 });
44894516 }
44904517 } else {
44914518 return await sendGenerationRequest(type, generate_data, { jsonSchema });
44924519 }
44934520 }
44944521
@@ -4520,6 +4547,12 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
45204547 throw new Error(data?.response);
45214548 }
45224549
4550+ if (jsonSchema) {
4551+ unblockGeneration(type);
4552+ generatedPromptCache = '';
4553+ return extractJsonFromData(data);
4554+ }
4555+
45234556 //const getData = await response.json();
45244557 let getMessage = extractMessageFromData(data);
45254558 let title = extractTitleFromData(data);
@@ -5097,15 +5130,21 @@ function setInContextMessages(msgInContextCount, type) {
50975130}
50985131
50995132/**
5133+ * @typedef {object} AdditionalRequestOptions
5134+ * @property {JsonSchema} [jsonSchema]
5135+ */
5136+
5137+/**
51005138 * Sends a non-streaming request to the API.
51015139 * @param {string} type Generation type
51025140 * @param {object} data Generation data
5141+ * @param {AdditionalRequestOptions} [options] Additional options for the generation request
51035142 * @returns {Promise<object>} Response data from the API
51045143 * @throws {Error|object}
51055144 */
51065145export async function sendGenerationRequest(type, data, options = {}) {
51075146 if (main_api === 'openai') {
51085147 return await sendOpenAIRequest(type, data.prompt, abortController.signal, options);
51095148 }
51105149
51115150 if (main_api === 'koboldhorde') {
@@ -5131,16 +5170,17 @@ export async function sendGenerationRequest(type, data) {
51315170 * Sends a streaming request to the API.
51325171 * @param {string} type Generation type
51335172 * @param {object} data Generation data
5173+ * @param {AdditionalRequestOptions} [options] Additional options for the generation request
51345174 * @returns {Promise<any>} Streaming generator
51355175 */
51365176export async function sendStreamingRequest(type, data, options = {}) {
51375177 if (abortController?.signal?.aborted) {
51385178 throw new Error('Generation was aborted.');
51395179 }
51405180
51415181 switch (main_api) {
51425182 case 'openai':
51435183 return await sendOpenAIRequest(type, data.prompt, streamingProcessor.abortController.signal, options);
51445184 case 'textgenerationwebui':
51455185 return await generateTextGenWithStreaming(data, streamingProcessor.abortController.signal);
51465186 case 'novel':
@@ -5282,6 +5322,58 @@ export function extractMessageFromData(data, activeApi = null) {
52825322}
52835323
52845324/**
5325+ * Extracts JSON from the response data.
5326+ * @param {object} data Response data
5327+ * @returns {string} Extracted JSON string from the response data
5328+ */
5329+export function extractJsonFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
5330+ mainApi = mainApi ?? main_api;
5331+ chatCompletionSource = chatCompletionSource ?? oai_settings.chat_completion_source;
5332+
5333+ const tryParse = (/** @type {string} */ value) => {
5334+ try {
5335+ return JSON.parse(value);
5336+ } catch (e) {
5337+ console.debug('Failed to parse content as JSON.', e);
5338+ }
5339+ };
5340+
5341+ let result = {};
5342+
5343+ switch (mainApi) {
5344+ case 'openai': {
5345+ const text = extractMessageFromData(data, mainApi);
5346+ switch (chatCompletionSource) {
5347+ case chat_completion_sources.CLAUDE:
5348+ result = data?.content?.find(x => x.type === 'tool_use')?.input;
5349+ break;
5350+ case chat_completion_sources.PERPLEXITY:
5351+ result = tryParse(removeReasoningFromString(text));
5352+ break;
5353+ case chat_completion_sources.VERTEXAI:
5354+ case chat_completion_sources.MAKERSUITE:
5355+ case chat_completion_sources.DEEPSEEK:
5356+ case chat_completion_sources.AI21:
5357+ case chat_completion_sources.GROQ:
5358+ case chat_completion_sources.POLLINATIONS:
5359+ case chat_completion_sources.AIMLAPI:
5360+ case chat_completion_sources.OPENAI:
5361+ case chat_completion_sources.OPENROUTER:
5362+ case chat_completion_sources.MISTRALAI:
5363+ case chat_completion_sources.CUSTOM:
5364+ case chat_completion_sources.COHERE:
5365+ case chat_completion_sources.XAI:
5366+ default:
5367+ result = tryParse(text);
5368+ break;
5369+ }
5370+ } break;
5371+ }
5372+
5373+ return JSON.stringify(result ?? {});
5374+}
5375+
5376+/**
52855377 * Extracts multiswipe swipes from the response data.
52865378 * @param {Object} data Response data
52875379 * @param {string} type Type of generation
public/scripts/custom-request.js+7 -2
@@ -1,5 +1,5 @@
11import { getPresetManager } from './preset-manager.js';
22import { extractJsonFromData, extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js';
33import { getTextGenServer } from './textgen-settings.js';
44import { extractReasoningFromData } from './reasoning.js';
55import { formatInstructModeChat, formatInstructModePrompt, getInstructStoppingSequences, names_behavior_types } from './instruct-mode.js';
@@ -467,7 +467,7 @@ export class ChatCompletionService {
467467 return json;
468468 }
469469
470470 returnconst result = {
471471 content: extractMessageFromData(json, this.TYPE),
472472 reasoning: extractReasoningFromData(json, {
473473 mainApi: this.TYPE,
@@ -475,6 +475,11 @@ export class ChatCompletionService {
475475 ignoreShowThoughts: true,
476476 }),
477477 };
478+ // Try parse JSON
479+ if (data.json_schema) {
480+ result.content = JSON.parse(extractJsonFromData(json, { mainApi: this.TYPE, chatCompletionSource: data.chat_completion_source }));
481+ }
482+ return result;
478483 }
479484
480485 if (!response.ok) {
public/scripts/openai.js+6 -1
@@ -2187,11 +2187,12 @@ function getReasoningEffort() {
21872187 * @param {string} type (impersonate, quiet, continue, etc)
21882188 * @param {Array} messages
21892189 * @param {AbortSignal?} signal
2190+ * @param {import('../script.js').AdditionalRequestOptions} options
21902191 * @returns {Promise<unknown>}
21912192 * @throws {Error}
21922193 */
21932194
21942195async function sendOpenAIRequest(type, messages, signal, { jsonSchema = null } = {}) {
21952196 // Provide default abort signal
21962197 if (!signal) {
21972198 signal = new AbortController().signal;
@@ -2463,6 +2464,10 @@ async function sendOpenAIRequest(type, messages, signal) {
24632464 }
24642465 }
24652466
2467+ if (jsonSchema) {
2468+ generate_data.json_schema = jsonSchema;
2469+ }
2470+
24662471 await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data);
24672472
24682473 const generate_url = '/api/backends/chat-completions/generate';
public/scripts/st-context.js+2 -0
@@ -52,6 +52,7 @@ import {
5252 getCharacterCardFields,
5353 swipe_right,
5454 swipe_left,
55+ generateRaw,
5556} from '../script.js';
5657import {
5758 extension_settings,
@@ -170,6 +171,7 @@ export function getContext() {
170171 ModuleWorkerWrapper,
171172 getTokenizerModel,
172173 generateQuietPrompt,
174+ generateRaw,
173175 writeExtensionField,
174176 getThumbnailUrl,
175177 selectCharacterById,
src/endpoints/backends/chat-completions.js+140 -3
@@ -18,6 +18,7 @@ import {
1818 excludeKeysByYaml,
1919 color,
2020 trimTrailingSlash,
21+ flattenSchema,
2122} from '../../util.js';
2223import {
2324 convertClaudeMessages,
@@ -173,6 +174,17 @@ async function sendClaudeRequest(request, response) {
173174 }
174175 }
175176
177+ // Structured output is a forced tool
178+ if (request.body.json_schema) {
179+ const jsonTool = {
180+ name: request.body.json_schema.name,
181+ description: request.body.json_schema.description || 'Well-formed JSON object',
182+ input_schema: request.body.json_schema.value,
183+ };
184+ requestBody.tools = [...(requestBody.tools || []), jsonTool];
185+ requestBody.tool_choice = { type: 'tool', name: request.body.json_schema.name };
186+ }
187+
176188 if (useWebSearch) {
177189 const webSearchTool = [{
178190 'type': 'web_search_20250305',
@@ -363,6 +375,9 @@ async function sendMakerSuiteRequest(request, response) {
363375 const isGemma = model.includes('gemma');
364376 const isLearnLM = model.includes('learnlm');
365377
378+ const responseMimeType = request.body.responseMimeType ?? (request.body.json_schema ? 'application/json' : undefined);
379+ const responseSchema = request.body.responseSchema ?? (request.body.json_schema ? request.body.json_schema.value : undefined);
380+
366381 const generationConfig = {
367382 stopSequences: request.body.stop,
368383 candidateCount: 1,
@@ -370,8 +385,8 @@ async function sendMakerSuiteRequest(request, response) {
370385 temperature: request.body.temperature,
371386 topP: request.body.top_p,
372387 topK: request.body.top_k || undefined,
373388 responseMimeType: request.body.responseMimeType,
374389 responseSchema: request.body.responseSchema,
375390 };
376391
377392 function getGeminiBody() {
@@ -615,12 +630,23 @@ async function sendAI21Request(request, response) {
615630 return response.status(400).send({ error: true });
616631 }
617632
633+ const bodyParams = {};
618634 const controller = new AbortController();
619- console.debug(request.body.messages);
620635 request.socket.removeAllListeners('close');
621636 request.socket.on('close', function () {
622637 controller.abort();
623638 });
639+ // Hack to support JSON schema
640+ if (request.body.json_schema) {
641+ bodyParams.response_format = {
642+ type: 'json_object',
643+ };
644+ const message = {
645+ role: 'user',
646+ content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
647+ };
648+ request.body.messages.push(message);
649+ }
624650 const convertedPrompt = convertAI21Messages(request.body.messages, getPromptNames(request));
625651 const body = {
626652 messages: convertedPrompt,
@@ -631,6 +657,7 @@ async function sendAI21Request(request, response) {
631657 stop: request.body.stop,
632658 stream: request.body.stream,
633659 tools: request.body.tools,
660+ ...bodyParams,
634661 };
635662 const options = {
636663 method: 'POST',
@@ -711,6 +738,18 @@ async function sendMistralAIRequest(request, response) {
711738 requestBody['tool_choice'] = request.body.tool_choice;
712739 }
713740
741+ if (request.body.json_schema) {
742+ requestBody['response_format'] = {
743+ type: 'json_schema',
744+ json_schema: {
745+ name: request.body.json_schema.name,
746+ description: request.body.json_schema.description,
747+ schema: request.body.json_schema.value,
748+ strict: request.body.json_schema.strict ?? true,
749+ },
750+ };
751+ }
752+
714753 const config = {
715754 method: 'POST',
716755 headers: {
@@ -801,6 +840,13 @@ async function sendCohereRequest(request, response) {
801840 requestBody.safety_mode = 'OFF';
802841 }
803842
843+ if (request.body.json_schema) {
844+ requestBody.response_format = {
845+ type: 'json_schema',
846+ schema: request.body.json_schema.value,
847+ };
848+ }
849+
804850 console.debug('Cohere request:', requestBody);
805851
806852 const config = {
@@ -882,6 +928,18 @@ async function sendDeepSeekRequest(request, response) {
882928 });
883929 }
884930
931+ // Hack to support JSON schema
932+ if (request.body.json_schema) {
933+ bodyParams.response_format = {
934+ type: 'json_object',
935+ };
936+ const message = {
937+ role: 'user',
938+ content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
939+ };
940+ request.body.messages.push(message);
941+ }
942+
885943 const postProcessType = String(request.body.model).endsWith('-reasoner')
886944 ? PROMPT_PROCESSING_TYPE.STRICT_TOOLS
887945 : PROMPT_PROCESSING_TYPE.SEMI_TOOLS;
@@ -990,6 +1048,17 @@ async function sendXaiRequest(request, response) {
9901048 };
9911049 }
9921050
1051+ if (request.body.json_schema) {
1052+ bodyParams['response_format'] = {
1053+ type: 'json_schema',
1054+ json_schema: {
1055+ name: request.body.json_schema.name,
1056+ strict: request.body.json_schema.strict ?? true,
1057+ schema: request.body.json_schema.value,
1058+ },
1059+ };
1060+ }
1061+
9931062 const processedMessages = request.body.messages = convertXAIMessages(request.body.messages, getPromptNames(request));
9941063
9951064 const requestBody = {
@@ -1085,6 +1154,18 @@ async function sendAimlapiRequest(request, response) {
10851154 bodyParams['reasoning_effort'] = request.body.reasoning_effort;
10861155 }
10871156
1157+ if (request.body.json_schema) {
1158+ bodyParams['response_format'] = {
1159+ type: 'json_schema',
1160+ json_schema: {
1161+ name: request.body.json_schema.name,
1162+ description: request.body.json_schema.description,
1163+ schema: request.body.json_schema.value,
1164+ strict: request.body.json_schema.strict ?? true,
1165+ },
1166+ };
1167+ }
1168+
10881169 const requestBody = {
10891170 'messages': request.body.messages,
10901171 'model': request.body.model,
@@ -1405,6 +1486,10 @@ router.post('/generate', function (request, response) {
14051486 getPromptNames(request));
14061487 }
14071488
1489+ if (request.body.json_schema?.value) {
1490+ request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
1491+ }
1492+
14081493 switch (request.body.chat_completion_source) {
14091494 case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response);
14101495 case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response);
@@ -1480,6 +1565,17 @@ router.post('/generate', function (request, response) {
14801565 bodyParams['reasoning'] = { effort: request.body.reasoning_effort };
14811566 }
14821567
1568+ if (request.body.json_schema) {
1569+ bodyParams['response_format'] = {
1570+ type: 'json_schema',
1571+ json_schema: {
1572+ name: request.body.json_schema.name,
1573+ strict: request.body.json_schema.strict ?? true,
1574+ schema: request.body.json_schema.value,
1575+ },
1576+ };
1577+ }
1578+
14831579 const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
14841580 const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4)/.test(request.body.model);
14851581 const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';
@@ -1516,11 +1612,30 @@ router.post('/generate', function (request, response) {
15161612 reasoning_effort: request.body.reasoning_effort,
15171613 };
15181614 request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
1615+ if (request.body.json_schema) {
1616+ bodyParams['response_format'] = {
1617+ type: 'json_schema',
1618+ json_schema: {
1619+ schema: request.body.json_schema.value,
1620+ },
1621+ };
1622+ }
15191623 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
15201624 apiUrl = API_GROQ;
15211625 apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
15221626 headers = {};
15231627 bodyParams = {};
1628+ if (request.body.json_schema) {
1629+ bodyParams['response_format'] = {
1630+ type: 'json_schema',
1631+ json_schema: {
1632+ name: request.body.json_schema.name,
1633+ description: request.body.json_schema.description,
1634+ schema: request.body.json_schema.value,
1635+ strict: request.body.json_schema.strict ?? true,
1636+ },
1637+ };
1638+ }
15241639 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
15251640 apiUrl = API_NANOGPT;
15261641 apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
@@ -1543,6 +1658,17 @@ router.post('/generate', function (request, response) {
15431658 referrer: 'sillytavern',
15441659 seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
15451660 };
1661+ // Hack to support JSON schema
1662+ if (request.body.json_schema) {
1663+ bodyParams['response_format'] = {
1664+ type: 'json_object',
1665+ };
1666+ const message = {
1667+ role: 'user',
1668+ content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
1669+ };
1670+ request.body.messages.push(message);
1671+ }
15461672 } else {
15471673 console.warn('This chat completion source is not supported yet.');
15481674 return response.status(400).send({ error: true });
@@ -1581,6 +1707,17 @@ router.post('/generate', function (request, response) {
15811707 bodyParams['tool_choice'] = request.body.tool_choice;
15821708 }
15831709
1710+ if (request.body.json_schema && !bodyParams['response_format']) {
1711+ bodyParams['response_format'] = {
1712+ type: 'json_schema',
1713+ json_schema: {
1714+ name: request.body.json_schema.name,
1715+ strict: request.body.json_schema.strict ?? true,
1716+ schema: request.body.json_schema.value,
1717+ },
1718+ };
1719+ }
1720+
15841721 const requestBody = {
15851722 'messages': isTextCompletion === false ? request.body.messages : undefined,
15861723 'prompt': isTextCompletion === true ? textPrompt : undefined,
src/util.js+63 -1
@@ -17,7 +17,7 @@ import mime from 'mime-types';
1717import { default as simpleGit } from 'simple-git';
1818import chalk from 'chalk';
1919import bytes from 'bytes';
2020import { LOG_LEVELS, CHAT_COMPLETION_SOURCES } from './constants.js';
2121import { serverDirectory } from './server-directory.js';
2222
2323/**
@@ -1213,3 +1213,65 @@ export function getRequestURL(request) {
12131213 }
12141214 throw new TypeError('Invalid request type');
12151215}
1216+
1217+/**
1218+ * Flattens a JSON schema by inlining all definitions and setting additionalProperties to false.
1219+ * @param {object} schema The JSON schema to flatten.
1220+ * @param {string} api The API source, used to determine how to handle certain properties.
1221+ * @returns {object} The flattened schema.
1222+ */
1223+export function flattenSchema(schema, api) {
1224+ if (!schema || typeof schema !== 'object') {
1225+ return schema;
1226+ }
1227+
1228+ // Deep clone to avoid modifying the original object.
1229+ const schemaCopy = structuredClone(schema);
1230+
1231+ const definitions = schemaCopy.$defs || {};
1232+ delete schemaCopy.$defs;
1233+
1234+ function replaceRefs(obj) {
1235+ if (obj === null || typeof obj !== 'object') {
1236+ return obj;
1237+ }
1238+
1239+ if (Array.isArray(obj)) {
1240+ for (let i = 0; i < obj.length; i++) {
1241+ obj[i] = replaceRefs(obj[i]);
1242+ }
1243+ return obj;
1244+ }
1245+
1246+ if (obj.$ref && typeof obj.$ref === 'string' && obj.$ref.startsWith('#/$defs/')) {
1247+ const defName = obj.$ref.split('/').pop();
1248+ if (definitions[defName]) {
1249+ return replaceRefs(structuredClone(definitions[defName]));
1250+ }
1251+ }
1252+
1253+ if (api === CHAT_COMPLETION_SOURCES.MAKERSUITE || api === CHAT_COMPLETION_SOURCES.VERTEXAI) {
1254+ delete obj.default;
1255+ delete obj.additionalProperties;
1256+ } else if ('properties' in obj) {
1257+ if (obj.additionalProperties === undefined || obj.additionalProperties === true) {
1258+ obj.additionalProperties = false;
1259+ }
1260+ }
1261+
1262+ for (const key in obj) {
1263+ if (Object.prototype.hasOwnProperty.call(obj, key)) {
1264+ obj[key] = replaceRefs(obj[key]);
1265+ }
1266+ }
1267+ return obj;
1268+ }
1269+
1270+ const flattenedSchema = replaceRefs(schemaCopy);
1271+
1272+ if (flattenedSchema.$schema) {
1273+ delete flattenedSchema.$schema;
1274+ }
1275+
1276+ return flattenedSchema;
1277+}