Added structured output for common APIs (#4272) * Added structured output for common APIs * eslint * Added frontend impl * Type name change * Unprefix json_schema, apply review suggestions * Add schema to generateQuietPrompt, add comments * Prettify diff * Extract JSON from Claude response * Add structured gen for Mistral * Hack to support schema for DeepSeek * Hack JSON schema for AI21 * Add Groq structured gen * Add JSON mode for pollinations * Add JSON schema for perplexity * Add JSON schema for AIML * Using extractJsonFromData in custom-request, added google rules for flattenSchema * Fix response parsing * Fix Google * Fixed json parse * Expose generateRaw to getContext --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

cd176039ef0fef85c74bb3315abc535aca912fc8

bmen25124 <bmen25124@gmail.com>

Signed

Parents: 87f8a3eb2c30d83bd52a7ebbbfeab3131162364b

public/script.js+111 -19 Viewed

 /**
  * Background generation based on the provided prompt.
  * @param {string} quiet_promptquietPrompt Instruction prompt for the AI
  * @param {boolean} [quietToLoud] Whether the message should be sent in a foreground (loud) or background (quiet) mode
  * @param {boolean} [skipWIAN] whetherWhether to skip addition of World Info and Author's Note into the prompt
  * @param {string} [quietImage] Image to use for the quiet prompt
  * @param {string} [quietName] Name to use for the quiet prompt (defaults to "System:")
  * @param {number} [responseLength] Maximum response length. If unset, the global default value is used.
  * @param {number} force_chid[forceChId] Character ID to use for this generation run. Works in groups only.
 - * @returns
 + * @param {AdditionalRequestOptions} [options={}] Additional generation request options.
 + * @returns {Promise<string>} Generated text. If using structured output, will contain a serialized JSON object.
  */
 export async function generateQuietPrompt(quiet_promptquietPrompt, quietToLoud = false, skipWIAN = false, quietImage = null, quietName = null, responseLength = null, force_chidforceChId = null, { jsonSchema } = {}) {
     console.log('got into genQuietPrompt');
     const responseLengthCustomized = typeof responseLength === 'number' && responseLength > 0;
     let eventHook = () => { };
     try {
         /** @type {GenerateOptions} */
         const options = {
             quiet_prompt: quietPrompt,
             quietToLoud,
             skipWIAN: skipWIAN,
             force_name2: true,
             quietImage: quietImage,
             quietName: quietName,
             force_chid: force_chidforceChId,
 +            jsonSchema: jsonSchema,
         };
         if (responseLengthCustomized) {
             TempResponseLength.save(main_api, responseLength);

  * @param {number} [responseLength] Maximum response length. If unset, the global default value is used.
  * @param {boolean} [trimNames] Whether to allow trimming "{{user}}:" and "{{char}}:" from the response.
  * @param {string} [prefill] An optional prefill for the prompt.
 + * @param {AdditionalRequestOptions} [options] Additional options for generation
  * @returns {Promise<string>} Generated message
  */
 export async function generateRaw(prompt, api, instructOverride, quietToLoud, systemPrompt, responseLength, trimNames = true, prefill = '', options = {}) {
     if (!api) {
         api = main_api;
+    }

         if (api === 'koboldhorde') {
             data = await generateHorde(prompt.toString(), generateData, abortController.signal, false);
         } else if (api === 'openai') {
             data = await sendOpenAIRequest('quiet', generateData, abortController.signal, options);
         } else {
             const generateUrl = getGenerateUrl(api);
             const response = await fetch(generateUrl, {

             throw new Error(data.response);
+        }
 +        if (options?.jsonSchema) {
 +            return extractJsonFromData(data, { mainApi: api });
 +        }
++
         // format result, exclude user prompt bias
         const message = cleanUpMessage({
             getMessage: extractMessageFromData(data),

+}
 /**
 + * @typedef {object} JsonSchema
 + * @property {string} name Name of the schema.
 + * @property {object} value JSON schema value.
 + * @property {string} [description] Description of the schema.
 + * @property {boolean} [strict] If true, the schema will be used in strict mode, meaning that only the fields defined in the schema will be allowed.
 + *
 + * @typedef {object} GenerateOptions
 + * @property {boolean} [automatic_trigger] If the generation was triggered automatically (e.g. group auto mode).
 + * @property {boolean} [force_name2] If a char name should be forced to add to the prompt's last line (Text Completion, non-Instruct only).
 + * @property {string} [quiet_prompt] A system instruction to use for the quiet prompt.
 + * @property {boolean} [quietToLoud] Whether the system instruction should be sent in background (quiet) or a foreground (loud) mode.
 + * @property {boolean} [skipWIAN] Skip adding World Info and Author's Note to the prompt.
 + * @property {number} [force_chid] Force character ID to use for the generation. Only works in groups.
 + * @property {AbortSignal} [signal] Abort signal to cancel the generation. If not provided, will create a new AbortController.
 + * @property {string} [quietImage] Image URL to use for the quiet prompt (defaults to empty string)
 + * @property {string} [quietName] Name to use for the quiet prompt (defaults to "System:")
 + * @property {number} [depth] Recursion depth for the generation. Used to prevent infinite loops in tool calls.
 + * @property {JsonSchema} [jsonSchema] JSON schema to use for the structured generation. Usually requires a special instruction.
 + */
++
 +/**
  * MARK:Generate()
  * Runs a generation using the current chat context.
  * @param {string} type Generation type
  * @param {GenerateOptions} options Generation options
  * @param {boolean} dryRun Whether to actually generate a message or just assemble the prompt
  * @returns {Promise<any>} Returns a promise that resolves when the text is done generating.
 - * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, quietName?: string, depth?: number }} GenerateOptions
  */
 export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, jsonSchema = null, depth = 0 } = {}, dryRun = false) {
     console.log('Generate entered');
     setGenerationProgress(0);
     generation_started = new Date();

@@ -4488,7 +4515,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
4488	4515	});
4489	4516	}
4490	4517	} else {
4491	4518	return await sendGenerationRequest(type, generate_data, { jsonSchema });
4492	4519	}
4493	4520	}
4494	4521

             throw new Error(data?.response);
+        }
 +        if (jsonSchema) {
 +            unblockGeneration(type);
 +            generatedPromptCache = '';
 +            return extractJsonFromData(data);
 +        }
++
         //const getData = await response.json();
         let getMessage = extractMessageFromData(data);
         let title = extractTitleFromData(data);

+}
 /**
 + * @typedef {object} AdditionalRequestOptions
 + * @property {JsonSchema} [jsonSchema]
 + */
++
 +/**
  * Sends a non-streaming request to the API.
  * @param {string} type Generation type
  * @param {object} data Generation data
 + * @param {AdditionalRequestOptions} [options] Additional options for the generation request
  * @returns {Promise<object>} Response data from the API
  * @throws {Error|object}
  */
 export async function sendGenerationRequest(type, data, options = {}) {
     if (main_api === 'openai') {
         return await sendOpenAIRequest(type, data.prompt, abortController.signal, options);
+    }
     if (main_api === 'koboldhorde') {

  * Sends a streaming request to the API.
  * @param {string} type Generation type
  * @param {object} data Generation data
 + * @param {AdditionalRequestOptions} [options] Additional options for the generation request
  * @returns {Promise<any>} Streaming generator
  */
 export async function sendStreamingRequest(type, data, options = {}) {
     if (abortController?.signal?.aborted) {
         throw new Error('Generation was aborted.');
+    }
     switch (main_api) {
         case 'openai':
             return await sendOpenAIRequest(type, data.prompt, streamingProcessor.abortController.signal, options);
         case 'textgenerationwebui':
             return await generateTextGenWithStreaming(data, streamingProcessor.abortController.signal);
         case 'novel':

+}
 /**
 + * Extracts JSON from the response data.
 + * @param {object} data Response data
 + * @returns {string} Extracted JSON string from the response data
 + */
 +export function extractJsonFromData(data, { mainApi = null, chatCompletionSource = null } = {}) {
 +    mainApi = mainApi ?? main_api;
 +    chatCompletionSource = chatCompletionSource ?? oai_settings.chat_completion_source;
++
 +    const tryParse = (/** @type {string} */ value) => {
 +        try {
 +            return JSON.parse(value);
 +        } catch (e) {
 +            console.debug('Failed to parse content as JSON.', e);
 +        }
 +    };
++
 +    let result = {};
++
 +    switch (mainApi) {
 +        case 'openai': {
 +            const text = extractMessageFromData(data, mainApi);
 +            switch (chatCompletionSource) {
 +                case chat_completion_sources.CLAUDE:
 +                    result = data?.content?.find(x => x.type === 'tool_use')?.input;
 +                    break;
 +                case chat_completion_sources.PERPLEXITY:
 +                    result = tryParse(removeReasoningFromString(text));
 +                    break;
 +                case chat_completion_sources.VERTEXAI:
 +                case chat_completion_sources.MAKERSUITE:
 +                case chat_completion_sources.DEEPSEEK:
 +                case chat_completion_sources.AI21:
 +                case chat_completion_sources.GROQ:
 +                case chat_completion_sources.POLLINATIONS:
 +                case chat_completion_sources.AIMLAPI:
 +                case chat_completion_sources.OPENAI:
 +                case chat_completion_sources.OPENROUTER:
 +                case chat_completion_sources.MISTRALAI:
 +                case chat_completion_sources.CUSTOM:
 +                case chat_completion_sources.COHERE:
 +                case chat_completion_sources.XAI:
 +                default:
 +                    result = tryParse(text);
 +                    break;
 +            }
 +        } break;
 +    }
++
 +    return JSON.stringify(result ?? {});
 +}
++
 +/**
  * Extracts multiswipe swipes from the response data.
  * @param {Object} data Response data
  * @param {string} type Type of generation

public/scripts/custom-request.js+7 -2 Viewed

 import { getPresetManager } from './preset-manager.js';
 import { extractJsonFromData, extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js';
 import { getTextGenServer } from './textgen-settings.js';
 import { extractReasoningFromData } from './reasoning.js';
 import { formatInstructModeChat, formatInstructModePrompt, getInstructStoppingSequences, names_behavior_types } from './instruct-mode.js';

                 return json;
+            }
             returnconst result = {
                 content: extractMessageFromData(json, this.TYPE),
                 reasoning: extractReasoningFromData(json, {
                     mainApi: this.TYPE,

                     ignoreShowThoughts: true,
                 }),
             };
 +            // Try parse JSON
 +            if (data.json_schema) {
 +                result.content = JSON.parse(extractJsonFromData(json, { mainApi: this.TYPE, chatCompletionSource: data.chat_completion_source }));
 +            }
 +            return result;
+        }
         if (!response.ok) {

public/scripts/openai.js+6 -1 Viewed

  * @param {string} type (impersonate, quiet, continue, etc)
  * @param {Array} messages
  * @param {AbortSignal?} signal
 + * @param {import('../script.js').AdditionalRequestOptions} options
  * @returns {Promise<unknown>}
  * @throws {Error}
  */
 async function sendOpenAIRequest(type, messages, signal, { jsonSchema = null } = {}) {
     // Provide default abort signal
     if (!signal) {
         signal = new AbortController().signal;

@@ -2463,6 +2464,10 @@ async function sendOpenAIRequest(type, messages, signal) {
2463	2464	}
2464	2465	}
2465	2466
	2467	+ if (jsonSchema) {
	2468	+ generate_data.json_schema = jsonSchema;
	2469	+ }
	2470	+
2466	2471	await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data);
2467	2472
2468	2473	const generate_url = '/api/backends/chat-completions/generate';

public/scripts/st-context.js+2 -0 Viewed

     getCharacterCardFields,
     swipe_right,
     swipe_left,
 +    generateRaw,
 } from '../script.js';
 import {
     extension_settings,

         ModuleWorkerWrapper,
         getTokenizerModel,
         generateQuietPrompt,
 +        generateRaw,
         writeExtensionField,
         getThumbnailUrl,
         selectCharacterById,

src/endpoints/backends/chat-completions.js+140 -3 Viewed

     excludeKeysByYaml,
     color,
     trimTrailingSlash,
 +    flattenSchema,
 } from '../../util.js';
 import {
     convertClaudeMessages,

+            }
+        }
 +        // Structured output is a forced tool
 +        if (request.body.json_schema) {
 +            const jsonTool = {
 +                name: request.body.json_schema.name,
 +                description: request.body.json_schema.description || 'Well-formed JSON object',
 +                input_schema: request.body.json_schema.value,
 +            };
 +            requestBody.tools = [...(requestBody.tools || []), jsonTool];
 +            requestBody.tool_choice = { type: 'tool', name: request.body.json_schema.name };
 +        }
++
         if (useWebSearch) {
             const webSearchTool = [{
                 'type': 'web_search_20250305',

     const isGemma = model.includes('gemma');
     const isLearnLM = model.includes('learnlm');
 +    const responseMimeType = request.body.responseMimeType ?? (request.body.json_schema ? 'application/json' : undefined);
 +    const responseSchema = request.body.responseSchema ?? (request.body.json_schema ? request.body.json_schema.value : undefined);
++
     const generationConfig = {
         stopSequences: request.body.stop,
         candidateCount: 1,

         temperature: request.body.temperature,
         topP: request.body.top_p,
         topK: request.body.top_k || undefined,
         responseMimeType: request.body.responseMimeType,
         responseSchema: request.body.responseSchema,
     };
     function getGeminiBody() {

         return response.status(400).send({ error: true });
+    }
 +    const bodyParams = {};
     const controller = new AbortController();
 -    console.debug(request.body.messages);
     request.socket.removeAllListeners('close');
     request.socket.on('close', function () {
         controller.abort();
     });
 +    // Hack to support JSON schema
 +    if (request.body.json_schema) {
 +        bodyParams.response_format = {
 +            type: 'json_object',
 +        };
 +        const message = {
 +            role: 'user',
 +            content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
 +        };
 +        request.body.messages.push(message);
 +    }
     const convertedPrompt = convertAI21Messages(request.body.messages, getPromptNames(request));
     const body = {
         messages: convertedPrompt,

         stop: request.body.stop,
         stream: request.body.stream,
         tools: request.body.tools,
 +        ...bodyParams,
     };
     const options = {
         method: 'POST',

             requestBody['tool_choice'] = request.body.tool_choice;
+        }
 +        if (request.body.json_schema) {
 +            requestBody['response_format'] = {
 +                type: 'json_schema',
 +                json_schema: {
 +                    name: request.body.json_schema.name,
 +                    description: request.body.json_schema.description,
 +                    schema: request.body.json_schema.value,
 +                    strict: request.body.json_schema.strict ?? true,
 +                },
 +            };
 +        }
++
         const config = {
             method: 'POST',
             headers: {

             requestBody.safety_mode = 'OFF';
+        }
 +        if (request.body.json_schema) {
 +            requestBody.response_format = {
 +                type: 'json_schema',
 +                schema: request.body.json_schema.value,
 +            };
 +        }
++
         console.debug('Cohere request:', requestBody);
         const config = {

             });
+        }
 +        // Hack to support JSON schema
 +        if (request.body.json_schema) {
 +            bodyParams.response_format = {
 +                type: 'json_object',
 +            };
 +            const message = {
 +                role: 'user',
 +                content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
 +            };
 +            request.body.messages.push(message);
 +        }
++
         const postProcessType = String(request.body.model).endsWith('-reasoner')
             ? PROMPT_PROCESSING_TYPE.STRICT_TOOLS
             : PROMPT_PROCESSING_TYPE.SEMI_TOOLS;

             };
+        }
 +        if (request.body.json_schema) {
 +            bodyParams['response_format'] = {
 +                type: 'json_schema',
 +                json_schema: {
 +                    name: request.body.json_schema.name,
 +                    strict: request.body.json_schema.strict ?? true,
 +                    schema: request.body.json_schema.value,
 +                },
 +            };
 +        }
++
         const processedMessages = request.body.messages = convertXAIMessages(request.body.messages, getPromptNames(request));
         const requestBody = {

             bodyParams['reasoning_effort'] = request.body.reasoning_effort;
+        }
 +        if (request.body.json_schema) {
 +            bodyParams['response_format'] = {
 +                type: 'json_schema',
 +                json_schema: {
 +                    name: request.body.json_schema.name,
 +                    description: request.body.json_schema.description,
 +                    schema: request.body.json_schema.value,
 +                    strict: request.body.json_schema.strict ?? true,
 +                },
 +            };
 +        }
++
         const requestBody = {
             'messages': request.body.messages,
             'model': request.body.model,

             getPromptNames(request));
+    }
 +    if (request.body.json_schema?.value) {
 +        request.body.json_schema.value = flattenSchema(request.body.json_schema.value, request.body.chat_completion_source);
 +    }
++
     switch (request.body.chat_completion_source) {
         case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response);
         case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response);

             bodyParams['reasoning'] = { effort: request.body.reasoning_effort };
+        }
 +        if (request.body.json_schema) {
 +            bodyParams['response_format'] = {
 +                type: 'json_schema',
 +                json_schema: {
 +                    name: request.body.json_schema.name,
 +                    strict: request.body.json_schema.strict ?? true,
 +                    schema: request.body.json_schema.value,
 +                },
 +            };
 +        }
++
         const cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1, 'number');
         const isClaude3or4 = /anthropic\/claude-(3|opus-4|sonnet-4)/.test(request.body.model);
         const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m';

             reasoning_effort: request.body.reasoning_effort,
         };
         request.body.messages = postProcessPrompt(request.body.messages, PROMPT_PROCESSING_TYPE.STRICT, getPromptNames(request));
 +        if (request.body.json_schema) {
 +            bodyParams['response_format'] = {
 +                type: 'json_schema',
 +                json_schema: {
 +                    schema: request.body.json_schema.value,
 +                },
 +            };
 +        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
         apiUrl = API_GROQ;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
         headers = {};
         bodyParams = {};
 +        if (request.body.json_schema) {
 +            bodyParams['response_format'] = {
 +                type: 'json_schema',
 +                json_schema: {
 +                    name: request.body.json_schema.name,
 +                    description: request.body.json_schema.description,
 +                    schema: request.body.json_schema.value,
 +                    strict: request.body.json_schema.strict ?? true,
 +                },
 +            };
 +        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.NANOGPT) {
         apiUrl = API_NANOGPT;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);

             referrer: 'sillytavern',
             seed: request.body.seed ?? Math.floor(Math.random() * 99999999),
         };
 +        // Hack to support JSON schema
 +        if (request.body.json_schema) {
 +            bodyParams['response_format'] = {
 +                type: 'json_object',
 +            };
 +            const message = {
 +                role: 'user',
 +                content: `JSON schema for the response:\n${JSON.stringify(request.body.json_schema.value, null, 4)}`,
 +            };
 +            request.body.messages.push(message);
 +        }
     } else {
         console.warn('This chat completion source is not supported yet.');
         return response.status(400).send({ error: true });

         bodyParams['tool_choice'] = request.body.tool_choice;
+    }
 +    if (request.body.json_schema && !bodyParams['response_format']) {
 +        bodyParams['response_format'] = {
 +            type: 'json_schema',
 +            json_schema: {
 +                name: request.body.json_schema.name,
 +                strict: request.body.json_schema.strict ?? true,
 +                schema: request.body.json_schema.value,
 +            },
 +        };
 +    }
++
     const requestBody = {
         'messages': isTextCompletion === false ? request.body.messages : undefined,
         'prompt': isTextCompletion === true ? textPrompt : undefined,

src/util.js+63 -1 Viewed

 import { default as simpleGit } from 'simple-git';
 import chalk from 'chalk';
 import bytes from 'bytes';
 import { LOG_LEVELS, CHAT_COMPLETION_SOURCES } from './constants.js';
 import { serverDirectory } from './server-directory.js';
 /**

+    }
     throw new TypeError('Invalid request type');
+}
++
 +/**
 + * Flattens a JSON schema by inlining all definitions and setting additionalProperties to false.
 + * @param {object} schema The JSON schema to flatten.
 + * @param {string} api The API source, used to determine how to handle certain properties.
 + * @returns {object} The flattened schema.
 + */
 +export function flattenSchema(schema, api) {
 +    if (!schema || typeof schema !== 'object') {
 +        return schema;
 +    }
++
 +    // Deep clone to avoid modifying the original object.
 +    const schemaCopy = structuredClone(schema);
++
 +    const definitions = schemaCopy.$defs || {};
 +    delete schemaCopy.$defs;
++
 +    function replaceRefs(obj) {
 +        if (obj === null || typeof obj !== 'object') {
 +            return obj;
 +        }
++
 +        if (Array.isArray(obj)) {
 +            for (let i = 0; i < obj.length; i++) {
 +                obj[i] = replaceRefs(obj[i]);
 +            }
 +            return obj;
 +        }
++
 +        if (obj.$ref && typeof obj.$ref === 'string' && obj.$ref.startsWith('#/$defs/')) {
 +            const defName = obj.$ref.split('/').pop();
 +            if (definitions[defName]) {
 +                return replaceRefs(structuredClone(definitions[defName]));
 +            }
 +        }
++
 +        if (api === CHAT_COMPLETION_SOURCES.MAKERSUITE || api === CHAT_COMPLETION_SOURCES.VERTEXAI) {
 +            delete obj.default;
 +            delete obj.additionalProperties;
 +        } else if ('properties' in obj) {
 +            if (obj.additionalProperties === undefined || obj.additionalProperties === true) {
 +                obj.additionalProperties = false;
 +            }
 +        }
++
 +        for (const key in obj) {
 +            if (Object.prototype.hasOwnProperty.call(obj, key)) {
 +                obj[key] = replaceRefs(obj[key]);
 +            }
 +        }
 +        return obj;
 +    }
++
 +    const flattenedSchema = replaceRefs(schemaCopy);
++
 +    if (flattenedSchema.$schema) {
 +        delete flattenedSchema.$schema;
 +    }
++
 +    return flattenedSchema;
 +}