Added structured output for common APIs (#4272) * Added structured output for common APIs * eslint * Added frontend impl * Type name change * Unprefix json_schema, apply review suggestions * Add schema to generateQuietPrompt, add comments * Prettify diff * Extract JSON from Claude response * Add structured gen for Mistral * Hack to support schema for DeepSeek * Hack JSON schema for AI21 * Add Groq structured gen * Add JSON mode for pollinations * Add JSON schema for perplexity * Add JSON schema for AIML * Using extractJsonFromData in custom-request, added google rules for flattenSchema * Fix response parsing * Fix Google * Fixed json parse * Expose generateRaw to getContext --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

cd176039ef0fef85c74bb3315abc535aca912fc8

bmen25124 <bmen25124@gmail.com>

Signed

Parents: 87f8a3eb2c30d83bd52a7ebbbfeab3131162364b

public/script.js+111 -19 Viewed

 /**
  * Background generation based on the provided prompt.
- * @param {string} quiet_prompt Instruction prompt for the AI
+ * @param {string} quietPrompt Instruction prompt for the AI
- * @param {boolean} quietToLoud Whether the message should be sent in a foreground (loud) or background (quiet) mode
+ * @param {boolean} [quietToLoud] Whether the message should be sent in a foreground (loud) or background (quiet) mode
- * @param {boolean} skipWIAN whether to skip addition of World Info and Author's Note into the prompt
+ * @param {boolean} [skipWIAN] Whether to skip addition of World Info and Author's Note into the prompt
- * @param {string} quietImage Image to use for the quiet prompt
+ * @param {string} [quietImage] Image to use for the quiet prompt
- * @param {string} quietName Name to use for the quiet prompt (defaults to "System:")
+ * @param {string} [quietName] Name to use for the quiet prompt (defaults to "System:")
  * @param {number} [responseLength] Maximum response length. If unset, the global default value is used.
- * @param {number} force_chid Character ID to use for this generation run. Works in groups only.
+ * @param {number} [forceChId] Character ID to use for this generation run. Works in groups only.
- * @returns
+ * @param {AdditionalRequestOptions} [options={}] Additional generation request options.
+ * @returns {Promise<string>} Generated text. If using structured output, will contain a serialized JSON object.
  */
-export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN, quietImage = null, quietName = null, responseLength = null, force_chid = null) {
+export async function generateQuietPrompt(quietPrompt, quietToLoud = false, skipWIAN = false, quietImage = null, quietName = null, responseLength = null, forceChId = null, { jsonSchema } = {}) {
     console.log('got into genQuietPrompt');
     const responseLengthCustomized = typeof responseLength === 'number' && responseLength > 0;
     let eventHook = () => { };
     try {
         /** @type {GenerateOptions} */
         const options = {
-            quiet_prompt,
+            quiet_prompt: quietPrompt,
             quietToLoud,
             skipWIAN: skipWIAN,
             force_name2: true,
             quietImage: quietImage,
             quietName: quietName,
-            force_chid: force_chid,
+            force_chid: forceChId,
+            jsonSchema: jsonSchema,
         };
         if (responseLengthCustomized) {
             TempResponseLength.save(main_api, responseLength);

  * @param {number} [responseLength] Maximum response length. If unset, the global default value is used.
  * @param {boolean} [trimNames] Whether to allow trimming "{{user}}:" and "{{char}}:" from the response.
  * @param {string} [prefill] An optional prefill for the prompt.
+ * @param {AdditionalRequestOptions} [options] Additional options for generation
  * @returns {Promise<string>} Generated message
  */
-export async function generateRaw(prompt, api, instructOverride, quietToLoud, systemPrompt, responseLength, trimNames = true, prefill = '') {
+export async function generateRaw(prompt, api, instructOverride, quietToLoud, systemPrompt, responseLength, trimNames = true, prefill = '', options = {}) {
     if (!api) {
         api = main_api;
+    }

         if (api === 'koboldhorde') {
             data = await generateHorde(prompt.toString(), generateData, abortController.signal, false);
         } else if (api === 'openai') {
-            data = await sendOpenAIRequest('quiet', generateData, abortController.signal);
+            data = await sendOpenAIRequest('quiet', generateData, abortController.signal, options);
         } else {
             const generateUrl = getGenerateUrl(api);
             const response = await fetch(generateUrl, {

             throw new Error(data.response);
+        }
+        if (options?.jsonSchema) {
+            return extractJsonFromData(data, { mainApi: api });
+        }
         // format result, exclude user prompt bias
         const message = cleanUpMessage({
             getMessage: extractMessageFromData(data),

+}
 /**
+ * @typedef {object} JsonSchema
+ * @property {string} name Name of the schema.
+ * @property {object} value JSON schema value.
+ * @property {string} [description] Description of the schema.
+ * @property {boolean} [strict] If true, the schema will be used in strict mode, meaning that only the fields defined in the schema will be allowed.
+ *
+ * @typedef {object} GenerateOptions
+ * @property {boolean} [automatic_trigger] If the generation was triggered automatically (e.g. group auto mode).
+ * @property {boolean} [force_name2] If a char name should be forced to add to the prompt's last line (Text Completion, non-Instruct only).
+ * @property {string} [quiet_prompt] A system instruction to use for the quiet prompt.
+ * @property {boolean} [quietToLoud] Whether the system instruction should be sent in background (quiet) or a foreground (loud) mode.
+ * @property {boolean} [skipWIAN] Skip adding World Info and Author's Note to the prompt.
+ * @property {number} [force_chid] Force character ID to use for the generation. Only works in groups.
+ * @property {AbortSignal} [signal] Abort signal to cancel the generation. If not provided, will create a new AbortController.
+ * @property {string} [quietImage] Image URL to use for the quiet prompt (defaults to empty string)
+ * @property {string} [quietName] Name to use for the quiet prompt (defaults to "System:")
+ * @property {number} [depth] Recursion depth for the generation. Used to prevent infinite loops in tool calls.
+ * @property {JsonSchema} [jsonSchema] JSON schema to use for the structured generation. Usually requires a special instruction.
+ */
+/**
  * MARK:Generate()
  * Runs a generation using the current chat context.
  * @param {string} type Generation type
  * @param {GenerateOptions} options Generation options
  * @param {boolean} dryRun Whether to actually generate a message or just assemble the prompt
  * @returns {Promise<any>} Returns a promise that resolves when the text is done generating.
- * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, quietName?: string, depth?: number }} GenerateOptions
  */
-export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, depth = 0 } = {}, dryRun = false) {
+export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, jsonSchema = null, depth = 0 } = {}, dryRun = false) {
     console.log('Generate entered');
     setGenerationProgress(0);
     generation_started = new Date();

@@ -4488,7 +4515,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
4488	});	4515	});
4489	}	4516	}
4490	} else {	4517	} else {
4491	return await sendGenerationRequest(type, generate_data);	4518	return await sendGenerationRequest(type, generate_data, { jsonSchema });
4492	}	4519	}
4493	}	4520	}
4494		4521

             throw new Error(data?.response);
+        }
+        if (jsonSchema) {
+            unblockGeneration(type);
+            generatedPromptCache = '';
+            return extractJsonFromData(data);
+        }
         //const getData = await response.json();
         let getMessage = extractMessageFromData(data);
         let title = extractTitleFromData(data);

+}
 /**
+ * @typedef {object} AdditionalRequestOptions
+ * @property {JsonSchema} [jsonSchema]
+ */
+/**
  * Sends a non-streaming request to the API.
  * @param {string} type Generation type
  * @param {object} data Generation data
+ * @param {AdditionalRequestOptions} [options] Additional options for the generation request
  * @returns {Promise<object>} Response data from the API
  * @throws {Error|object}
  */
-export async function sendGenerationRequest(type, data) {
+export async function sendGenerationRequest(type, data, options = {}) {
     if (main_api === 'openai') {
-        return await sendOpenAIRequest(type, data.prompt, abortController.signal);
+        return await sendOpenAIRequest(type, data.prompt, abortController.signal, options);
+    }
     if (main_api === 'koboldhorde') {

  * Sends a streaming request to the API.
  * @param {string} type Generation type
  * @param {object} data Generation data
+ * @param {AdditionalRequestOptions} [options] Additional options for the generation request
  * @returns {Promise<any>} Streaming generator
  */
-export async function sendStreamingRequest(type, data) {
+export async function sendStreamingRequest(type, data, options = {}) {
     if (abortController?.signal?.aborted) {
         throw new Error('Generation was aborted.');
+    }
     switch (main_api) {
         case 'openai':
-            return await sendOpenAIRequest(type, data.prompt, streamingProcessor.abortController.signal);
+            return await sendOpenAIRequest(type, data.prompt, streamingProcessor.abortController.signal, options);
         case 'textgenerationwebui':
             return await generateTextGenWithStreaming(data, streamingProcessor.abortController.signal);
         case 'novel':

+}
 /**
+ * Extracts JSON from the response data.
+ * @param {object} data Response data
+ * @returns {string} Extracted JSON string from the response data
+ */
+export function extractJsonFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
+    mainApi = mainApi ?? main_api;
+    chatCompletionSource = chatCompletionSource ?? oai_settings.chat_completion_source;
+    const tryParse = (/** @type {string} */ value) => {
+        try {
+            return JSON.parse(value);
+        } catch (e) {
+            console.debug('Failed to parse content as JSON.', e);
+        }
+    };
+    let result = {};
+    switch (mainApi) {
+        case 'openai': {
+            const text = extractMessageFromData(data, mainApi);
+            switch (chatCompletionSource) {
+                case chat_completion_sources.CLAUDE:
+                    result = data?.content?.find(x => x.type === 'tool_use')?.input;
+                    break;
+                case chat_completion_sources.PERPLEXITY:
+                    result = tryParse(removeReasoningFromString(text));
+                    break;
+                case chat_completion_sources.VERTEXAI:
+                case chat_completion_sources.MAKERSUITE:
+                case chat_completion_sources.DEEPSEEK:
+                case chat_completion_sources.AI21:
+                case chat_completion_sources.GROQ:
+                case chat_completion_sources.POLLINATIONS:
+                case chat_completion_sources.AIMLAPI:
+                case chat_completion_sources.OPENAI:
+                case chat_completion_sources.OPENROUTER:
+                case chat_completion_sources.MISTRALAI:
+                case chat_completion_sources.CUSTOM:
+                case chat_completion_sources.COHERE:
+                case chat_completion_sources.XAI:
+                default:
+                    result = tryParse(text);
+                    break;
+            }
+        } break;
+    }
+    return JSON.stringify(result ?? {});
+}
+/**
  * Extracts multiswipe swipes from the response data.
  * @param {Object} data Response data
  * @param {string} type Type of generation

public/scripts/custom-request.js+7 -2 Viewed

 import { getPresetManager } from './preset-manager.js';
-import { extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js';
+import { extractJsonFromData, extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js';
 import { getTextGenServer } from './textgen-settings.js';
 import { extractReasoningFromData } from './reasoning.js';
 import { formatInstructModeChat, formatInstructModePrompt, getInstructStoppingSequences, names_behavior_types } from './instruct-mode.js';

                 return json;
+            }
-            return {
+            const result = {
                 content: extractMessageFromData(json, this.TYPE),
                 reasoning: extractReasoningFromData(json, {
                     mainApi: this.TYPE,

                     ignoreShowThoughts: true,
                 }),
             };
+            // Try parse JSON
+            if (data.json_schema) {
+                result.content = JSON.parse(extractJsonFromData(json, { mainApi: this.TYPE, chatCompletionSource: data.chat_completion_source }));
+            }
+            return result;
+        }
         if (!response.ok) {

public/scripts/openai.js+6 -1 Viewed

  * @param {string} type (impersonate, quiet, continue, etc)
  * @param {Array} messages
  * @param {AbortSignal?} signal
+ * @param {import('../script.js').AdditionalRequestOptions} options
  * @returns {Promise<unknown>}
  * @throws {Error}
  */
-async function sendOpenAIRequest(type, messages, signal) {
+async function sendOpenAIRequest(type, messages, signal, { jsonSchema = null } = {}) {
     // Provide default abort signal
     if (!signal) {
         signal = new AbortController().signal;

@@ -2463,6 +2464,10 @@ async function sendOpenAIRequest(type, messages, signal) {
2463	}	2464	}
2464	}	2465	}
2465		2466
		2467	if (jsonSchema) {
		2468	generate_data.json_schema = jsonSchema;
		2469	}
		2470
2466	await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data);	2471	await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data);
2467		2472
2468	const generate_url = '/api/backends/chat-completions/generate';	2473	const generate_url = '/api/backends/chat-completions/generate';

public/scripts/st-context.js+2 -0 Viewed

     getCharacterCardFields,
     swipe_right,
     swipe_left,
+    generateRaw,
 } from '../script.js';
 import {
     extension_settings,

         ModuleWorkerWrapper,
         getTokenizerModel,
         generateQuietPrompt,
+        generateRaw,
         writeExtensionField,
         getThumbnailUrl,
         selectCharacterById,

src/endpoints/backends/chat-completions.js+140 -3 Viewed

     excludeKeysByYaml,
     color,
     trimTrailingSlash,
+    flattenSchema,
 } from '../../util.js';
 import {
     convertClaudeMessages,

+            }
+        }
+        // Structured output is a forced tool
+        if (request.body.json_schema) {
+            const jsonTool = {
+                name: request.body.json_schema.name,
+                description: request.body.json_schema.description || 'Well-formed JSON object',
+                input_schema: request.body.json_schema.value,
+            };
+            requestBody.tools = [...(requestBody.tools || []), jsonTool];
+            requestBody.tool_choice = { type: 'tool', name: request.body.json_schema.name };
+        }
         if (useWebSearch) {
             const webSearchTool = [{
                 'type': 'web_search_20250305',

     const isGemma = model.includes('gemma');
     const isLearnLM = model.includes('learnlm');
+    const responseMimeType = request.body.responseMimeType ?? (request.body.json_schema ? 'application/json' : undefined);
+    const responseSchema = request.body.responseSchema ?? (request.body.json_schema ? request.body.json_schema.value : undefined);
     const generationConfig = {
         stopSequences: request.body.stop,
         candidateCount: 1,

         temperature: request.body.temperature,
         topP: request.body.top_p,
         topK: request.body.top_k || undefined,
-        responseMimeType: request.body.responseMimeType,
+        responseMimeType: responseMimeType,
-        responseSchema: request.body.responseSchema,
+        responseSchema: responseSchema,
     };
     function getGeminiBody() {

         return response.status(400).send({ error: true });
+    }
+    const bodyParams = {};
     const controller = new AbortController();
-    console.debug(request.body.messages);
     request.socket.removeAllListeners('close');
     request.socket.on('close', function () {
         controller.abort();
     });
+    // Hack to support JSON schema
+    if (request.body.json_schema) {
+        bodyParams.response_format = {
+            type: 'json_object',
+        };
+        const message = {
+            role: 'user',
+            content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
+        };
+        request.body.messages.push(message);
+    }
     const convertedPrompt = convertAI21Messages(request.body.messages, getPromptNames(request));
     const body = {
         messages: convertedPrompt,

         stop: request.body.stop,
         stream: request.body.stream,
         tools: request.body.tools,
+        ...bodyParams,
     };
     const options = {
         method: 'POST',

             requestBody['tool_choice'] = request.body.tool_choice;
+        }
+        if (request.body.json_schema) {
+            requestBody['response_format'] = {
+                type: 'json_schema',
+                json_schema: {
+                    name: request.body.json_schema.name,
+                    description: request.body.json_schema.description,
+                    schema: request.body.json_schema.value,
+                    strict: request.body.json_schema.strict ?? true,
+                },
+            };
+        }
         const config = {
             method: 'POST',
             headers: {

             requestBody.safety_mode = 'OFF';
+        }
+        if (request.body.json_schema) {
+            requestBody.response_format = {
+                type: 'json_schema',
+                schema: request.body.json_schema.value,
+            };
+        }
         console.debug('Cohere request:', requestBody);
         const config = {

             });
+        }
+        // Hack to support JSON schema
+        if (request.body.json_schema) {
+            bodyParams.response_format = {
+                type: 'json_object',
+            };
+            const message = {
+                role: 'user',
+                content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
+            };
+            request.body.messages.push(message);
+        }
         const postProcessType = String(request.body.model).endsWith('-reasoner')
             ? PROMPT_PROCESSING_TYPE.STRICT_TOOLS
             : PROMPT_PROCESSING_TYPE.SEMI_TOOLS;

             };
+        }
+        if (request.body.json_schema) {
+            bodyParams['response_format'] = {
+                type: 'json_schema',
+                json_schema: {
+                    name: request.body.json_schema.name,
+                    strict: request.body.json_schema.strict ?? true,
+                    schema: request.body.json_schema.value,
+                },
+            };
+        }
         const processedMessages = request.body.messages = convertXAIMessages(request.body.messages, getPromptNames(request));
         const requestBody = {

             bodyParams['reasoning_effort'] = request.body.reasoning_effort;
+        }
+        if (request.body.json_schema) {
+            bodyParams['response_format'] = {
+                type: 'json_schema',
+                json_schema: {
+                    name: request.body.json_schema.name,
+                    description: request.body.json_schema.description,
+                    schema: request.body.json_schema.value,
+                    strict: request.body.json_schema.strict ?? true,
+                },
+            };
+        }
         const requestBody = {
             'messages': request.body.messages,
             'model': request.body.model,

             getPromptNames(request));
+    }
+    if (request.body.json_schema?.value) {
+        request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
+    }
     switch (request.body.chat_completion_source) {
         case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response);
         case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response);

             bodyParams['reasoning'] = { effort: request.body.reasoning_effort };
+        }
+        if (request.body.json_schema) {
+            bodyParams['response_format'] = {
+                type: 'json_schema',
+                json_schema: {
+                    name: request.body.json_schema.name,
+                    strict: request.body.json_schema.strict ?? true,
+                    schema: request.body.json_schema.value,
+                },
+            };
+        }
         const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
         const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4)/.test(request.body.model);
         const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';

             reasoning_effort: request.body.reasoning_effort,
         };
         request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
+        if (request.body.json_schema) {
+            bodyParams['response_format'] = {
+                type: 'json_schema',
+                json_schema: {
+                    schema: request.body.json_schema.value,
+                },
+            };
+        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
         apiUrl = API_GROQ;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
         headers = {};
         bodyParams = {};
+        if (request.body.json_schema) {
+            bodyParams['response_format'] = {
+                type: 'json_schema',
+                json_schema: {
+                    name: request.body.json_schema.name,
+                    description: request.body.json_schema.description,
+                    schema: request.body.json_schema.value,
+                    strict: request.body.json_schema.strict ?? true,
+                },
+            };
+        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
         apiUrl = API_NANOGPT;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);

             referrer: 'sillytavern',
             seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
         };
+        // Hack to support JSON schema
+        if (request.body.json_schema) {
+            bodyParams['response_format'] = {
+                type: 'json_object',
+            };
+            const message = {
+                role: 'user',
+                content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
+            };
+            request.body.messages.push(message);
+        }
     } else {
         console.warn('This chat completion source is not supported yet.');
         return response.status(400).send({ error: true });

         bodyParams['tool_choice'] = request.body.tool_choice;
+    }
+    if (request.body.json_schema && !bodyParams['response_format']) {
+        bodyParams['response_format'] = {
+            type: 'json_schema',
+            json_schema: {
+                name: request.body.json_schema.name,
+                strict: request.body.json_schema.strict ?? true,
+                schema: request.body.json_schema.value,
+            },
+        };
+    }
     const requestBody = {
         'messages': isTextCompletion === false ? request.body.messages : undefined,
         'prompt': isTextCompletion === true ? textPrompt : undefined,

src/util.js+63 -1 Viewed

 import { default as simpleGit } from 'simple-git';
 import chalk from 'chalk';
 import bytes from 'bytes';
-import { LOG_LEVELS } from './constants.js';
+import { LOG_LEVELS, CHAT_COMPLETION_SOURCES } from './constants.js';
 import { serverDirectory } from './server-directory.js';
 /**

+    }
     throw new TypeError('Invalid request type');
+}
+/**
+ * Flattens a JSON schema by inlining all definitions and setting additionalProperties to false.
+ * @param {object} schema The JSON schema to flatten.
+ * @param {string} api The API source, used to determine how to handle certain properties.
+ * @returns {object} The flattened schema.
+ */
+export function flattenSchema(schema, api) {
+    if (!schema || typeof schema !== 'object') {
+        return schema;
+    }
+    // Deep clone to avoid modifying the original object.
+    const schemaCopy = structuredClone(schema);
+    const definitions = schemaCopy.$defs || {};
+    delete schemaCopy.$defs;
+    function replaceRefs(obj) {
+        if (obj === null || typeof obj !== 'object') {
+            return obj;
+        }
+        if (Array.isArray(obj)) {
+            for (let i = 0; i < obj.length; i++) {
+                obj[i] = replaceRefs(obj[i]);
+            }
+            return obj;
+        }
+        if (obj.$ref && typeof obj.$ref === 'string' && obj.$ref.startsWith('#/$defs/')) {
+            const defName = obj.$ref.split('/').pop();
+            if (definitions[defName]) {
+                return replaceRefs(structuredClone(definitions[defName]));
+            }
+        }
+        if (api === CHAT_COMPLETION_SOURCES.MAKERSUITE || api === CHAT_COMPLETION_SOURCES.VERTEXAI) {
+            delete obj.default;
+            delete obj.additionalProperties;
+        } else if ('properties' in obj) {
+            if (obj.additionalProperties === undefined || obj.additionalProperties === true) {
+                obj.additionalProperties = false;
+            }
+        }
+        for (const key in obj) {
+            if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                obj[key] = replaceRefs(obj[key]);
+            }
+        }
+        return obj;
+    }
+    const flattenedSchema = replaceRefs(schemaCopy);
+    if (flattenedSchema.$schema) {
+        delete flattenedSchema.$schema;
+    }
+    return flattenedSchema;
+}