Implement Gemini thought signatures (#4886) * Implement Gemini thought signatures * Implement streaming support for Gemini thought signatures * Implement OR support for Gemini thought signatures * Remove unnecessary extraction of thought sigs from response parts * Update thought sig comments to remove explicit Gemini mention * Fix thought_signature naming convention in message.extra * Add thought_signatures to ReasoningMessageExtra typedef * Prevent thought sigs being sent to incompatible endpoints * Move signatures to populateChatHistory, update for consistent casing * Code clean-up * Only send thought signatures if target model and API match original * Implement content-hash thought signature mapping * Change the data model + split for text/functions * Don't include signature to invocations if the model doesn't match * Fix function description * Remove misleading comment * Handle OpenRouter signatures * Improve message extra types * Prevent modifying original invocations when removing signatures * Fix return of openrouter non-streaming signatures * Remove redundant array check --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>
Signed| @@ -6,7 +6,8 @@ import { oai_settings } from './scripts/openai'; | ||
| 6 | 6 | import { textgenerationwebui_settings } from './scripts/textgen-settings'; |
| 7 | 7 | import { FileAttachment } from './scripts/chats'; |
| 8 | 8 | import { ReasoningMessageExtra } from './scripts/reasoning'; |
| 9 | 9 | import { IGNORE_SYMBOL, OVERSWIPE_BEHAVIOR } from './scripts/constants'; |
| 10 | +import { ToolInvocation } from './scripts/tool-calling'; | |
| 10 | 11 | |
| 11 | 12 | declare global { |
| 12 | 13 | // Custom types |
| @@ -83,13 +84,16 @@ declare global { | ||
| 83 | 84 | } |
| 84 | 85 | |
| 85 | 86 | interface BaseMessageExtra { |
| 87 | + api?: string; | |
| 88 | + model?: string; | |
| 89 | + type?: string; | |
| 86 | 90 | gen_id?: number; |
| 87 | 91 | bias?: string; |
| 88 | 92 | uses_system_ui?: boolean; |
| 89 | 93 | memory?: string; |
| 90 | 94 | display_text?: string; |
| 91 | 95 | reasoning_display_text?: string; |
| 92 | 96 | tool_invocations?: anyToolInvocation[]; |
| 93 | 97 | title?: string; |
| 94 | 98 | isSmallSys?: boolean; |
| 95 | 99 | token_count?: number; |
| @@ -115,6 +119,8 @@ declare global { | ||
| 115 | 119 | generationType?: number; |
| 116 | 120 | /** @deprecated Use `MediaAttachment.negative` instead */ |
| 117 | 121 | negative?: string; |
| 122 | + /** Will exclude this message from prompt processing */ | |
| 123 | + [IGNORE_SYMBOL]?: boolean; | |
| 118 | 124 | } |
| 119 | 125 | |
| 120 | 126 | type MediaAttachment = MediaAttachmentProps & ImageGenerationAttachmentProps & ImageCaptionAttachmentProps; |
| @@ -267,7 +267,7 @@ import { initServerHistory } from './scripts/server-history.js'; | ||
| 267 | 267 | import { initSettingsSearch } from './scripts/setting-search.js'; |
| 268 | 268 | import { initBulkEdit } from './scripts/bulk-edit.js'; |
| 269 | 269 | import { getContext } from './scripts/st-context.js'; |
| 270 | 270 | import { extractReasoningFromData, extractReasoningSignatureFromData, initReasoning, parseReasoningInSwipes, PromptReasoning, ReasoningHandler, removeReasoningFromString, updateReasoningUI } from './scripts/reasoning.js'; |
| 271 | 271 | import { accountStorage } from './scripts/util/AccountStorage.js'; |
| 272 | 272 | import { initWelcomeScreen, openPermanentAssistantChat, openPermanentAssistantCard, getPermanentAssistantAvatar } from './scripts/welcome-screen.js'; |
| 273 | 273 | import { initDataMaid } from './scripts/data-maid.js'; |
| @@ -3388,6 +3388,8 @@ class StreamingProcessor { | ||
| 3388 | 3388 | this.promptReasoning = promptReasoning; |
| 3389 | 3389 | /** @type {string[]} */ |
| 3390 | 3390 | this.images = []; |
| 3391 | + /** @type {string?} */ | |
| 3392 | + this.reasoningSignature = null; | |
| 3391 | 3393 | } |
| 3392 | 3394 | |
| 3393 | 3395 | /** |
| @@ -3581,6 +3583,12 @@ class StreamingProcessor { | ||
| 3581 | 3583 | appendMediaToMessage(message, $(this.messageDom)); |
| 3582 | 3584 | } |
| 3583 | 3585 | |
| 3586 | + // Store reasoning signature for models that support multi-turn context | |
| 3587 | + if (this.reasoningSignature) { | |
| 3588 | + message.extra = message.extra || {}; | |
| 3589 | + message.extra.reasoning_signature = this.reasoningSignature; | |
| 3590 | + } | |
| 3591 | + | |
| 3584 | 3592 | this.markUIGenStopped(); |
| 3585 | 3593 | |
| 3586 | 3594 | if (this.type !== 'impersonate') { |
| @@ -3675,6 +3683,7 @@ class StreamingProcessor { | ||
| 3675 | 3683 | // Get the updated reasoning string into the handler |
| 3676 | 3684 | this.reasoningHandler.updateReasoning(this.messageId, state?.reasoning); |
| 3677 | 3685 | this.images = state?.images ?? []; |
| 3686 | + this.reasoningSignature = state?.signature ?? null; | |
| 3678 | 3687 | await eventSource.emit(event_types.STREAM_TOKEN_RECEIVED, text); |
| 3679 | 3688 | await sw.tick(async () => await this.onProgressStreaming(this.messageId, this.continueMessage + text)); |
| 3680 | 3689 | } |
| @@ -5237,6 +5246,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro | ||
| 5237 | 5246 | let title = extractTitleFromData(data); |
| 5238 | 5247 | let reasoning = extractReasoningFromData(data); |
| 5239 | 5248 | let imageUrls = extractImagesFromData(data); |
| 5249 | + const reasoningSignature = extractReasoningSignatureFromData(data); | |
| 5240 | 5250 | kobold_horde_model = title; |
| 5241 | 5251 | |
| 5242 | 5252 | const swipes = extractMultiSwipes(data, type); |
| @@ -5280,10 +5290,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro | ||
| 5280 | 5290 | else { |
| 5281 | 5291 | // Without streaming we'll be having a full message on continuation. Treat it as a last chunk. |
| 5282 | 5292 | if (originalType !== 'continue') { |
| 5283 | 5293 | ({ type, getMessage } = await saveReply({ type, getMessage, title, swipes, reasoning, imageUrls, reasoningSignature })); |
| 5284 | 5294 | } |
| 5285 | 5295 | else { |
| 5286 | 5296 | ({ type, getMessage } = await saveReply({ type: 'appendFinal', getMessage, title, swipes, reasoning, imageUrls, reasoningSignature })); |
| 5287 | 5297 | } |
| 5288 | 5298 | |
| 5289 | 5299 | // This relies on `saveReply` having been called to add the message to the chat, so it must be last. |
| @@ -6332,16 +6342,17 @@ async function processImageAttachment(message, { imageUrls }) { | ||
| 6332 | 6342 | * @property {string[]} [swipes] Extra swipes |
| 6333 | 6343 | * @property {string} [reasoning] Message reasoning |
| 6334 | 6344 | * @property {string[]} [imageUrls] Links to images |
| 6345 | + * @property {string?} [reasoningSignature] Encrypted signature of the reasoning text | |
| 6335 | 6346 | * |
| 6336 | 6347 | * @typedef {object} SaveReplyResult |
| 6337 | 6348 | * @property {string} type Type of generation |
| 6338 | 6349 | * @property {string} getMessage Generated message |
| 6339 | 6350 | */ |
| 6340 | 6351 | export async function saveReply({ type, getMessage, fromStreaming = false, title = '', swipes = [], reasoning = '', imageUrls = [], reasoningSignature = null }) { |
| 6341 | 6352 | // Backward compatibility |
| 6342 | 6353 | if (arguments.length > 1 && typeof arguments[0] !== 'object') { |
| 6343 | 6354 | console.trace('saveReply called with positional arguments. Please use an object instead.'); |
| 6344 | 6355 | [type, getMessage, fromStreaming, title, swipes, reasoning, imageUrls, reasoningSignature] = arguments; |
| 6345 | 6356 | } |
| 6346 | 6357 | |
| 6347 | 6358 | if (type != 'append' && type != 'continue' && type != 'appendFinal' && chat.length && (chat[chat.length - 1]['swipe_id'] === undefined || |
| @@ -6377,6 +6388,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title | ||
| 6377 | 6388 | chat[chat.length - 1]['extra']['model'] = getGeneratingModel(); |
| 6378 | 6389 | chat[chat.length - 1]['extra']['reasoning'] = reasoning; |
| 6379 | 6390 | chat[chat.length - 1]['extra']['reasoning_duration'] = null; |
| 6391 | + chat[chat.length - 1]['extra']['reasoning_signature'] = reasoningSignature; | |
| 6380 | 6392 | await processImageAttachment(chat[chat.length - 1], { imageUrls }); |
| 6381 | 6393 | if (power_user.message_token_count_enabled) { |
| 6382 | 6394 | const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes']; |
| @@ -6401,6 +6413,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title | ||
| 6401 | 6413 | chat[chat.length - 1]['extra']['model'] = getGeneratingModel(); |
| 6402 | 6414 | chat[chat.length - 1]['extra']['reasoning'] = reasoning; |
| 6403 | 6415 | chat[chat.length - 1]['extra']['reasoning_duration'] = null; |
| 6416 | + chat[chat.length - 1]['extra']['reasoning_signature'] = reasoningSignature; | |
| 6404 | 6417 | await processImageAttachment(chat[chat.length - 1], { imageUrls }); |
| 6405 | 6418 | if (power_user.message_token_count_enabled) { |
| 6406 | 6419 | const tokenCountText = (reasoning || '') + chat[chat.length - 1]['mes']; |
| @@ -6421,6 +6434,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title | ||
| 6421 | 6434 | chat[chat.length - 1]['extra']['api'] = getGeneratingApi(); |
| 6422 | 6435 | chat[chat.length - 1]['extra']['model'] = getGeneratingModel(); |
| 6423 | 6436 | chat[chat.length - 1]['extra']['reasoning'] += reasoning; |
| 6437 | + chat[chat.length - 1]['extra']['reasoning_signature'] = reasoningSignature; | |
| 6424 | 6438 | await processImageAttachment(chat[chat.length - 1], { imageUrls }); |
| 6425 | 6439 | // We don't know if the reasoning duration extended, so we don't update it here on purpose. |
| 6426 | 6440 | if (power_user.message_token_count_enabled) { |
| @@ -6443,6 +6457,7 @@ export async function saveReply({ type, getMessage, fromStreaming = false, title | ||
| 6443 | 6457 | chat[chat.length - 1]['extra']['model'] = getGeneratingModel(); |
| 6444 | 6458 | chat[chat.length - 1]['extra']['reasoning'] = reasoning; |
| 6445 | 6459 | chat[chat.length - 1]['extra']['reasoning_duration'] = null; |
| 6460 | + chat[chat.length - 1]['extra']['reasoning_signature'] = reasoningSignature; | |
| 6446 | 6461 | if (power_user.trim_spaces) { |
| 6447 | 6462 | getMessage = getMessage.trim(); |
| 6448 | 6463 | } |
| @@ -517,13 +517,17 @@ async function validateReverseProxy() { | ||
| 517 | 517 | |
| 518 | 518 | /** |
| 519 | 519 | * Formats chat messages into chat completion messages. |
| 520 | 520 | * @param {objectChatMessage[]} chat - Array containing all messages. |
| 521 | 521 | * @returns {object[]} - Array containing all messages formatted for chat completion. |
| 522 | 522 | */ |
| 523 | 523 | function setOpenAIMessages(chat) { |
| 524 | 524 | let j = 0; |
| 525 | 525 | // clean openai msgs |
| 526 | 526 | const messages = []; |
| 527 | + // Get current API and model for thought signature validation | |
| 528 | + const currentApi = oai_settings.chat_completion_source; | |
| 529 | + const currentModel = getChatCompletionModel(); | |
| 530 | + | |
| 527 | 531 | for (let i = chat.length - 1; i >= 0; i--) { |
| 528 | 532 | let role = chat[j]['is_user'] ? 'user' : 'assistant'; |
| 529 | 533 | let content = chat[j]['mes']; |
| @@ -567,8 +571,26 @@ function setOpenAIMessages(chat) { | ||
| 567 | 571 | const media = chat[j]?.extra?.media; |
| 568 | 572 | const mediaDisplay = getMediaDisplay(chat[j]); |
| 569 | 573 | const mediaIndex = getMediaIndex(chat[j]); |
| 570 | 574 | const invocations = chat[j]?.extra?.tool_invocations?.slice(); |
| 571 | - messages[i] = { 'role': role, 'content': content, name: name, 'media': media, 'mediaDisplay': mediaDisplay, 'mediaIndex': mediaIndex, 'invocations': invocations }; | |
| 575 | + | |
| 576 | + // Only send thought signatures if they were generated by the same API and model | |
| 577 | + const originApi = chat[j]?.extra?.api; | |
| 578 | + const originModel = chat[j]?.extra?.model; | |
| 579 | + const isSameModel = originApi === currentApi && originModel === currentModel; | |
| 580 | + const signature = isSameModel ? chat[j]?.extra?.reasoning_signature : null; | |
| 581 | + | |
| 582 | + // Remove signatures from invocations if the API/model don't match | |
| 583 | + if (Array.isArray(invocations) && invocations.length > 0) { | |
| 584 | + invocations.forEach((invocation, index) => { | |
| 585 | + if (invocation.signature && !isSameModel) { | |
| 586 | + const cloneInvocation = structuredClone(invocation); | |
| 587 | + delete cloneInvocation.signature; | |
| 588 | + invocations[index] = cloneInvocation; | |
| 589 | + } | |
| 590 | + }); | |
| 591 | + } | |
| 592 | + | |
| 593 | + messages[i] = { 'role': role, 'content': content, name: name, 'media': media, 'mediaDisplay': mediaDisplay, 'mediaIndex': mediaIndex, 'invocations': invocations, 'signature': signature }; | |
| 572 | 594 | j++; |
| 573 | 595 | } |
| 574 | 596 | |
| @@ -863,6 +885,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul | ||
| 863 | 885 | const videoInlining = isVideoInliningSupported(); |
| 864 | 886 | const audioInlining = isAudioInliningSupported(); |
| 865 | 887 | const canUseTools = ToolManager.isToolCallingSupported(); |
| 888 | + const includeSignature = isReasoningSignatureSupported(); | |
| 866 | 889 | |
| 867 | 890 | // Insert chat messages as long as there is budget available |
| 868 | 891 | const chatPool = [...messages].reverse(); |
| @@ -918,7 +941,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul | ||
| 918 | 941 | const invocations = chatPrompt.invocations; |
| 919 | 942 | const toolCallMessage = await Message.createAsync(chatMessage.role, undefined, 'toolCall-' + chatMessage.identifier); |
| 920 | 943 | const toolResultMessages = await Promise.all(invocations.slice().reverse().map((invocation) => Message.createAsync('tool', invocation.result || '[No content]', invocation.id))); |
| 921 | 944 | await toolCallMessage.setToolCalls(invocations, includeSignature); |
| 922 | 945 | if (chatCompletion.canAffordAll([toolCallMessage, ...toolResultMessages])) { |
| 923 | 946 | for (const resultMessage of toolResultMessages) { |
| 924 | 947 | chatCompletion.insertAtStart(resultMessage, 'chatHistory'); |
| @@ -931,6 +954,10 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul | ||
| 931 | 954 | continue; |
| 932 | 955 | } |
| 933 | 956 | |
| 957 | + if (includeSignature && chatPrompt.signature) { | |
| 958 | + chatMessage.signature = chatPrompt.signature; | |
| 959 | + } | |
| 960 | + | |
| 934 | 961 | if (chatCompletion.canAfford(chatMessage)) { |
| 935 | 962 | chatCompletion.insertAtStart(chatMessage, 'chatHistory'); |
| 936 | 963 | } else { |
| @@ -2778,7 +2805,7 @@ async function sendOpenAIRequest(type, messages, signal, { jsonSchema = null } = | ||
| 2778 | 2805 | let text = ''; |
| 2779 | 2806 | const swipes = []; |
| 2780 | 2807 | const toolCalls = []; |
| 2781 | 2808 | const state = { reasoning: '', images: [], signature: '', toolSignatures: {} }; |
| 2782 | 2809 | while (true) { |
| 2783 | 2810 | const { done, value } = await reader.read(); |
| 2784 | 2811 | if (done) return; |
| @@ -2795,7 +2822,7 @@ async function sendOpenAIRequest(type, messages, signal, { jsonSchema = null } = | ||
| 2795 | 2822 | text += getStreamingReply(parsed, state); |
| 2796 | 2823 | } |
| 2797 | 2824 | |
| 2798 | 2825 | ToolManager.parseToolCalls(toolCalls, parsed, state.toolSignatures); |
| 2799 | 2826 | |
| 2800 | 2827 | yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed), toolCalls: toolCalls, state: state }; |
| 2801 | 2828 | } |
| @@ -2850,6 +2877,13 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov | ||
| 2850 | 2877 | if (show_thoughts) { |
| 2851 | 2878 | state.reasoning += (data?.candidates?.[0]?.content?.parts?.filter(x => x.thought)?.map(x => x.text)?.[0] || ''); |
| 2852 | 2879 | } |
| 2880 | + // Extract thought signatures from streaming chunks (typically in final chunk) | |
| 2881 | + const parts = data?.candidates?.[0]?.content?.parts || []; | |
| 2882 | + parts.forEach((part) => { | |
| 2883 | + if (part.thoughtSignature && typeof part.text === 'string') { | |
| 2884 | + state.signature = part.thoughtSignature; | |
| 2885 | + } | |
| 2886 | + }); | |
| 2853 | 2887 | return data?.candidates?.[0]?.content?.parts?.filter(x => !x.thought)?.map(x => x.text)?.[0] || ''; |
| 2854 | 2888 | } else if (chat_completion_source === chat_completion_sources.COHERE) { |
| 2855 | 2889 | return data?.delta?.message?.content?.text || data?.delta?.message?.tool_plan || ''; |
| @@ -2871,6 +2905,17 @@ export function getStreamingReply(data, state, { chatCompletionSource = null, ov | ||
| 2871 | 2905 | if (show_thoughts) { |
| 2872 | 2906 | state.reasoning += (data.choices?.filter(x => x?.delta?.reasoning)?.[0]?.delta?.reasoning || ''); |
| 2873 | 2907 | } |
| 2908 | + // Extract thought signatures from OpenRouter streaming (reasoning_details in delta) | |
| 2909 | + const reasoningDetails = data?.choices?.[0]?.delta?.reasoning_details || []; | |
| 2910 | + reasoningDetails.forEach((detail) => { | |
| 2911 | + if (detail.type === 'reasoning.encrypted' && detail.data) { | |
| 2912 | + if (/^tool_/.test(detail.id)) { | |
| 2913 | + state.toolSignatures[detail.id] = detail.data; | |
| 2914 | + } else { | |
| 2915 | + state.signature = detail.data; | |
| 2916 | + } | |
| 2917 | + } | |
| 2918 | + }); | |
| 2874 | 2919 | return data.choices?.[0]?.delta?.content ?? data.choices?.[0]?.message?.content ?? data.choices?.[0]?.text ?? ''; |
| 2875 | 2920 | } else if ([chat_completion_sources.CUSTOM, chat_completion_sources.POLLINATIONS, chat_completion_sources.AIMLAPI, chat_completion_sources.MOONSHOT, chat_completion_sources.COMETAPI, chat_completion_sources.ELECTRONHUB, chat_completion_sources.NANOGPT, chat_completion_sources.ZAI, chat_completion_sources.SILICONFLOW, chat_completion_sources.CHUTES].includes(chat_completion_source)) { |
| 2876 | 2921 | if (show_thoughts) { |
| @@ -3109,6 +3154,8 @@ class Message { | ||
| 3109 | 3154 | name; |
| 3110 | 3155 | /** @type {object} */ |
| 3111 | 3156 | tool_call = null; |
| 3157 | + /** @type {string?} */ | |
| 3158 | + signature = null; | |
| 3112 | 3159 | |
| 3113 | 3160 | /** |
| 3114 | 3161 | * @constructor |
| @@ -3150,9 +3197,10 @@ class Message { | ||
| 3150 | 3197 | /** |
| 3151 | 3198 | * Reconstruct the message from a tool invocation. |
| 3152 | 3199 | * @param {import('./tool-calling.js').ToolInvocation[]} invocations - The tool invocations to reconstruct the message from. |
| 3200 | + * @param {boolean} includeSignature Whether to include the signature in the tool calls. | |
| 3153 | 3201 | * @returns {Promise<void>} |
| 3154 | 3202 | */ |
| 3155 | 3203 | async setToolCalls(invocations, includeSignature) { |
| 3156 | 3204 | this.tool_calls = invocations.map(i => ({ |
| 3157 | 3205 | id: i.id, |
| 3158 | 3206 | type: 'function', |
| @@ -3160,6 +3208,7 @@ class Message { | ||
| 3160 | 3208 | arguments: i.parameters, |
| 3161 | 3209 | name: i.name, |
| 3162 | 3210 | }, |
| 3211 | + ...(includeSignature && i.signature ? { signature: i.signature } : {}), | |
| 3163 | 3212 | })); |
| 3164 | 3213 | this.tokens = await tokenHandler.countAsync({ role: this.role, tool_calls: JSON.stringify(this.tool_calls) }); |
| 3165 | 3214 | } |
| @@ -3411,6 +3460,7 @@ class MessageCollection { | ||
| 3411 | 3460 | ...(message.name && { name: message.name }), |
| 3412 | 3461 | ...(message.tool_calls && { tool_calls: message.tool_calls }), |
| 3413 | 3462 | ...(message.role === 'tool' && { tool_call_id: message.identifier }), |
| 3463 | + ...(message.signature && { signature: message.signature }), | |
| 3414 | 3464 | }); |
| 3415 | 3465 | } |
| 3416 | 3466 | return acc; |
| @@ -3702,6 +3752,7 @@ export class ChatCompletion { | ||
| 3702 | 3752 | ...(item.name ? { name: item.name } : {}), |
| 3703 | 3753 | ...(item.tool_calls ? { tool_calls: item.tool_calls } : {}), |
| 3704 | 3754 | ...(item.role === 'tool' ? { tool_call_id: item.identifier } : {}), |
| 3755 | + ...(item.signature ? { signature: item.signature } : {}), | |
| 3705 | 3756 | }; |
| 3706 | 3757 | chat.push(message); |
| 3707 | 3758 | } else { |
| @@ -5873,6 +5924,19 @@ export function isAudioInliningSupported() { | ||
| 5873 | 5924 | } |
| 5874 | 5925 | |
| 5875 | 5926 | /** |
| 5927 | + * Check if the model supports encrypted reasoning signatures. | |
| 5928 | + * @param {ChatCompletionSettings} settings Settings object to use | |
| 5929 | + * @returns {boolean} True if reasoning signatures should be included in the request | |
| 5930 | + */ | |
| 5931 | +export function isReasoningSignatureSupported(settings = oai_settings) { | |
| 5932 | + // If it's Vertex AI or Makersuite, that's OK - convertGooglePrompt() will handle it later | |
| 5933 | + const isGoogle = [chat_completion_sources.VERTEXAI, chat_completion_sources.MAKERSUITE].includes(settings.chat_completion_source); | |
| 5934 | + // Need a more crunchy check for OpenRouter: look for Gemini models | |
| 5935 | + const isOpenRouterGemini = settings.chat_completion_source === chat_completion_sources.OPENROUTER && /google\/gemini/i.test(settings.openrouter_model); | |
| 5936 | + return isGoogle || isOpenRouterGemini; | |
| 5937 | +} | |
| 5938 | + | |
| 5939 | +/** | |
| 5876 | 5940 | * Proxy stuff |
| 5877 | 5941 | */ |
| 5878 | 5942 | export function loadProxyPresets(settings) { |
| @@ -145,6 +145,53 @@ export function extractReasoningFromData(data, { | ||
| 145 | 145 | } |
| 146 | 146 | |
| 147 | 147 | /** |
| 148 | + * Extracts encrypted reasoning signature from the response data. | |
| 149 | + * These signatures are used to maintain reasoning context across multi-turn conversations. | |
| 150 | + * @param {object} data Response data | |
| 151 | + * @param {object} [options] Optional parameters | |
| 152 | + * @param {string|null} [options.mainApi] Override for main API | |
| 153 | + * @param {string|null} [options.chatCompletionSource] Override for chat completion source | |
| 154 | + * @returns {string?} Encrypted signature of the reasoning text | |
| 155 | + */ | |
| 156 | +export function extractReasoningSignatureFromData(data, { | |
| 157 | + mainApi = null, | |
| 158 | + chatCompletionSource = null, | |
| 159 | +} = {}) { | |
| 160 | + // Only Gemini models use thought signatures (via MakerSuite/VertexAI or OpenRouter) | |
| 161 | + if ((mainApi ?? main_api) !== 'openai') { | |
| 162 | + return null; | |
| 163 | + } | |
| 164 | + | |
| 165 | + const source = chatCompletionSource ?? oai_settings.chat_completion_source; | |
| 166 | + const isGemini = source === chat_completion_sources.MAKERSUITE || source === chat_completion_sources.VERTEXAI; | |
| 167 | + const isOpenRouter = source === chat_completion_sources.OPENROUTER; | |
| 168 | + | |
| 169 | + if (!isGemini && !isOpenRouter) { | |
| 170 | + return null; | |
| 171 | + } | |
| 172 | + | |
| 173 | + // OpenRouter format: reasoning_details array with type "reasoning.encrypted" (exclude tool calls) | |
| 174 | + if (isOpenRouter && Array.isArray(data?.choices?.[0]?.message?.reasoning_details)) { | |
| 175 | + for (const detail of data.choices[0].message.reasoning_details) { | |
| 176 | + if (!/^tool_/.test(detail.id) && detail.type === 'reasoning.encrypted' && detail.data) { | |
| 177 | + return detail.data; | |
| 178 | + } | |
| 179 | + } | |
| 180 | + } | |
| 181 | + | |
| 182 | + // Direct Gemini format: Extract from responseContent.parts if available (only text parts) | |
| 183 | + if (isGemini && Array.isArray(data?.responseContent?.parts)) { | |
| 184 | + data.responseContent.parts.forEach((part) => { | |
| 185 | + if (part.thoughtSignature && typeof part.text === 'string') { | |
| 186 | + return part.thoughtSignature; | |
| 187 | + } | |
| 188 | + }); | |
| 189 | + } | |
| 190 | + | |
| 191 | + return null; | |
| 192 | +} | |
| 193 | + | |
| 194 | +/** | |
| 148 | 195 | * Check if the model supports reasoning, but does not send back the reasoning |
| 149 | 196 | * @returns {boolean} True if the model supports reasoning |
| 150 | 197 | */ |
| @@ -1296,6 +1343,7 @@ export function parseReasoningFromString(str, { strict = true } = {}, template = | ||
| 1296 | 1343 | * @property {string} reasoning Reasoning block |
| 1297 | 1344 | * @property {number} reasoning_duration Duration of the reasoning block |
| 1298 | 1345 | * @property {string} reasoning_type Type of reasoning block |
| 1346 | + * @property {string?} reasoning_signature Encrypted signature of the reasoning text | |
| 1299 | 1347 | */ |
| 1300 | 1348 | export function parseReasoningInSwipes(swipes, swipeInfoArray, duration) { |
| 1301 | 1349 | if (!power_user.reasoning.auto_parse) { |
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | import { DOMPurify } from '../lib.js'; |
| 2 | 2 | |
| 3 | 3 | import { addOneMessage, chat, event_types, eventSource, getGeneratingApi, getGeneratingModel, main_api, saveChatConditional, system_avatar, systemUserName } from '../script.js'; |
| 4 | 4 | import { chat_completion_sources, custom_prompt_post_processing_types, getChatCompletionModel, model_list, oai_settings } from './openai.js'; |
| 5 | 5 | import { Popup } from './popup.js'; |
| 6 | 6 | import { SlashCommand } from './slash-commands/SlashCommand.js'; |
| @@ -19,6 +19,7 @@ import { isTrueBoolean } from './utils.js'; | ||
| 19 | 19 | * @property {string} name - The name of the tool. |
| 20 | 20 | * @property {string} parameters - The parameters for the tool invocation. |
| 21 | 21 | * @property {string} result - The result of the tool invocation. |
| 22 | + * @property {string?} signature - The thought signature associated with the tool invocation. | |
| 22 | 23 | */ |
| 23 | 24 | |
| 24 | 25 | /** |
| @@ -418,9 +419,10 @@ export class ToolManager { | ||
| 418 | 419 | * Utility function to parse tool calls from a parsed response. |
| 419 | 420 | * @param {any[]} toolCalls The tool calls to update. |
| 420 | 421 | * @param {any} parsed The parsed response from the OpenAI API. |
| 422 | + * @param {object} toolSignatures Optional mapping of tool call IDs to thought signatures. | |
| 421 | 423 | * @returns {void} |
| 422 | 424 | */ |
| 423 | 425 | static parseToolCalls(toolCalls, parsed, toolSignatures = {}) { |
| 424 | 426 | if (!this.isToolCallingSupported()) { |
| 425 | 427 | return; |
| 426 | 428 | } |
| @@ -457,6 +459,11 @@ export class ToolManager { | ||
| 457 | 459 | const targetToolCall = toolCalls[choiceIndex][toolCallIndex]; |
| 458 | 460 | |
| 459 | 461 | ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta); |
| 462 | + | |
| 463 | + // Transfer thought signature if available | |
| 464 | + if (Object.hasOwn(toolSignatures, targetToolCall.id)) { | |
| 465 | + targetToolCall.signature = toolSignatures[targetToolCall.id]; | |
| 466 | + } | |
| 460 | 467 | } |
| 461 | 468 | } |
| 462 | 469 | } |
| @@ -537,6 +544,9 @@ export class ToolManager { | ||
| 537 | 544 | toolCalls[choiceIndex][toolCallIndex] = {}; |
| 538 | 545 | } |
| 539 | 546 | const targetToolCall = toolCalls[choiceIndex][toolCallIndex]; |
| 547 | + if (part.thoughtSignature) { | |
| 548 | + targetToolCall.thoughtSignature = part.thoughtSignature; | |
| 549 | + } | |
| 540 | 550 | ToolManager.#applyToolCallDelta(targetToolCall, part.functionCall); |
| 541 | 551 | } |
| 542 | 552 | } |
| @@ -680,7 +690,7 @@ export class ToolManager { | ||
| 680 | 690 | const isClaudeToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isClaudeToolCall) : c?.input && c?.name && c?.id; |
| 681 | 691 | const isGoogleToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isGoogleToolCall) : c?.name && c?.args; |
| 682 | 692 | const convertClaudeToolCall = c => ({ id: c.id, function: { name: c.name, arguments: c.input } }); |
| 683 | 693 | const convertGoogleToolCall = (c, signature = null) => ({ id: getRandomId(), function: { name: c.name, arguments: c.args }, signature }); |
| 684 | 694 | |
| 685 | 695 | // Parsed tool calls from streaming data |
| 686 | 696 | if (Array.isArray(data) && data.length > 0 && Array.isArray(data[0])) { |
| @@ -689,7 +699,7 @@ export class ToolManager { | ||
| 689 | 699 | } |
| 690 | 700 | |
| 691 | 701 | if (isGoogleToolCall(data[0])) { |
| 692 | 702 | return data[0].filter(x => x).map((c) => convertGoogleToolCall(c, c.thoughtSignature)); |
| 693 | 703 | } |
| 694 | 704 | |
| 695 | 705 | if (typeof data[0]?.[0]?.tool_calls === 'object') { |
| @@ -701,7 +711,7 @@ export class ToolManager { | ||
| 701 | 711 | |
| 702 | 712 | // Google AI Studio tool calls |
| 703 | 713 | if (Array.isArray(data?.responseContent?.parts)) { |
| 704 | 714 | return data.responseContent.parts.filter(p => p.functionCall).map(p => convertGoogleToolCall(p.functionCall, p.thoughtSignature)); |
| 705 | 715 | } |
| 706 | 716 | |
| 707 | 717 | // Parsed tool calls from non-streaming data |
| @@ -709,7 +719,17 @@ export class ToolManager { | ||
| 709 | 719 | // Find a choice with 0-index |
| 710 | 720 | const choice = data.choices.find(choice => choice.index === 0); |
| 711 | 721 | |
| 712 | - if (choice) { | |
| 722 | + if (choice && typeof choice.message === 'object' && Array.isArray(choice.message.tool_calls)) { | |
| 723 | + // Add OpenRouter signatures | |
| 724 | + if (Array.isArray(choice.message.reasoning_details)) { | |
| 725 | + for (const toolCall of choice.message.tool_calls) { | |
| 726 | + const reasoningDetail = choice.message.reasoning_details.find(rd => rd.id === toolCall.id); | |
| 727 | + if (reasoningDetail && reasoningDetail.type === 'reasoning.encrypted' && reasoningDetail.data) { | |
| 728 | + toolCall.signature = reasoningDetail.data; | |
| 729 | + } | |
| 730 | + } | |
| 731 | + } | |
| 732 | + | |
| 713 | 733 | return choice.message.tool_calls; |
| 714 | 734 | } |
| 715 | 735 | } |
| @@ -792,6 +812,7 @@ export class ToolManager { | ||
| 792 | 812 | name, |
| 793 | 813 | parameters: stringify(parameters), |
| 794 | 814 | result: toolResult, |
| 815 | + signature: toolCall.signature || null, | |
| 795 | 816 | }; |
| 796 | 817 | result.invocations.push(invocation); |
| 797 | 818 | } |
| @@ -853,6 +874,8 @@ export class ToolManager { | ||
| 853 | 874 | extra: { |
| 854 | 875 | isSmallSys: true, |
| 855 | 876 | tool_invocations: invocations, |
| 877 | + api: getGeneratingApi(), | |
| 878 | + model: getGeneratingModel(), | |
| 856 | 879 | }, |
| 857 | 880 | }; |
| 858 | 881 | chat.push(message); |
| @@ -46,6 +46,7 @@ import { | ||
| 46 | 46 | embedOpenRouterMedia, |
| 47 | 47 | addReasoningContentToToolCalls, |
| 48 | 48 | cachingSystemPromptForOpenRouterClaude, |
| 49 | + addOpenRouterSignatures, | |
| 49 | 50 | } from '../../prompt-converters.js'; |
| 50 | 51 | |
| 51 | 52 | import { readSecret, SECRET_KEYS } from '../secrets.js'; |
| @@ -645,7 +646,7 @@ async function sendMakerSuiteRequest(request, response) { | ||
| 645 | 646 | return response.send({ error: { message } }); |
| 646 | 647 | } |
| 647 | 648 | |
| 648 | 649 | // Wrap it back to OAI format (responseContent includes thought signatures in parts array) |
| 649 | 650 | const reply = { choices: [{ 'message': { 'content': responseText } }], responseContent }; |
| 650 | 651 | return response.send(reply); |
| 651 | 652 | } |
| @@ -2048,6 +2049,7 @@ router.post('/generate', async function (request, response) { | ||
| 2048 | 2049 | const cacheTTL = getConfigValue('claude.extendedTTL', false, 'boolean') ? '1h' : '5m'; |
| 2049 | 2050 | if (Array.isArray(request.body.messages)) { |
| 2050 | 2051 | embedOpenRouterMedia(request.body.messages); |
| 2052 | + addOpenRouterSignatures(request.body.messages, request.body.model); | |
| 2051 | 2053 | |
| 2052 | 2054 | if (isClaude3or4) { |
| 2053 | 2055 | if (enableSystemPromptCache) { |
| @@ -548,6 +548,7 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) { | ||
| 548 | 548 | name: toolCall.function.name, |
| 549 | 549 | args: tryParse(toolCall.function.arguments) ?? toolCall.function.arguments, |
| 550 | 550 | }, |
| 551 | + ...(toolCall.signature ? { thoughtSignature: toolCall.signature } : {}), | |
| 551 | 552 | }); |
| 552 | 553 | |
| 553 | 554 | toolNameMap[toolCall.id] = toolCall.function.name; |
| @@ -567,17 +568,28 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) { | ||
| 567 | 568 | }); |
| 568 | 569 | |
| 569 | 570 | // https://ai.google.dev/gemini-api/docs/gemini-3#migrating_from_other_models |
| 570 | - if (/gemini-3/.test(model)) { | |
| 571 | + // Inject stored thought signatures, or fall back to bypass magic for Gemini 3 | |
| 572 | + if (/gemini-3/.test(model) || /gemini-2\.5/.test(model)) { | |
| 571 | 573 | const skipSignatureMagic = 'skip_thought_signature_validator'; |
| 572 | - parts.filter(p => p.functionCall).forEach(p => { | |
| 574 | + const textSignature = message.signature; | |
| 573 | - p.thoughtSignature = skipSignatureMagic; | |
| 575 | + | |
| 574 | - }); | |
| 576 | + parts.forEach((part) => { | |
| 577 | + if (textSignature && typeof part.text === 'string') { | |
| 578 | + part.thoughtSignature = textSignature; | |
| 579 | + } else if (/gemini-3/.test(model)) { | |
| 580 | + // Gemini 3: Fall back to bypass magic for function calls (mandatory) and images | |
| 581 | + if (part.functionCall && !part.thoughtSignature) { | |
| 582 | + part.thoughtSignature = skipSignatureMagic; | |
| 583 | + } | |
| 575 | 584 | if (/-image/.test(model) && message.role === 'model') { |
| 576 | 585 | parts.filter(p => if (typeof ppart.text === 'string' || ppart.inlineData).forEach(p => { |
| 577 | 586 | p part.thoughtSignature = skipSignatureMagic; |
| 578 | - }); | |
| 579 | 587 | } |
| 580 | 588 | } |
| 589 | + } | |
| 590 | + // Gemini 2.5 without stored signatures: signatures are optional, no bypass needed | |
| 591 | + }); | |
| 592 | + } | |
| 581 | 593 | |
| 582 | 594 | // merge consecutive messages with the same role |
| 583 | 595 | if (index > 0 && message.role === contents[contents.length - 1].role) { |
| @@ -1291,3 +1303,63 @@ export function addReasoningContentToToolCalls(messages) { | ||
| 1291 | 1303 | message.reasoning_content = ''; |
| 1292 | 1304 | } |
| 1293 | 1305 | } |
| 1306 | + | |
| 1307 | +/** | |
| 1308 | + * Converts reasoning signatures to OpenRouter format. | |
| 1309 | + * @param {object[]} messages Array of messages | |
| 1310 | + * @param {string} model Model name | |
| 1311 | + * @return {void} | |
| 1312 | + */ | |
| 1313 | +export function addOpenRouterSignatures(messages, model) { | |
| 1314 | + const getFormatForModel = () => { | |
| 1315 | + if (/google\/gemini/.test(model)) { | |
| 1316 | + return 'google-gemini-v1'; | |
| 1317 | + } | |
| 1318 | + if (/anthropic\/claude/.test(model)) { | |
| 1319 | + return 'anthropic-claude-v1'; | |
| 1320 | + } | |
| 1321 | + if (/openai\/gpt/.test(model)) { | |
| 1322 | + return 'openai-responses-v1'; | |
| 1323 | + } | |
| 1324 | + if (/x-ai\/grok/.test(model)) { | |
| 1325 | + return 'xai-responses-v1'; | |
| 1326 | + } | |
| 1327 | + return 'unknown'; | |
| 1328 | + }; | |
| 1329 | + | |
| 1330 | + if (!Array.isArray(messages)) { | |
| 1331 | + return; | |
| 1332 | + } | |
| 1333 | + | |
| 1334 | + for (const message of messages) { | |
| 1335 | + const details = []; | |
| 1336 | + const addDetail = (data, id) => { | |
| 1337 | + if (typeof data !== 'string' || data.length === 0) { | |
| 1338 | + return; | |
| 1339 | + } | |
| 1340 | + const detail = { | |
| 1341 | + index: details.length, | |
| 1342 | + id: id || `signature-${details.length}`, | |
| 1343 | + type: 'reasoning.encrypted', | |
| 1344 | + data: data, | |
| 1345 | + format: getFormatForModel(), | |
| 1346 | + }; | |
| 1347 | + details.push(detail); | |
| 1348 | + }; | |
| 1349 | + if (typeof message.signature === 'string') { | |
| 1350 | + addDetail(message.signature); | |
| 1351 | + delete message.signature; | |
| 1352 | + } | |
| 1353 | + if (Array.isArray(message.tool_calls)) { | |
| 1354 | + message.tool_calls.forEach((toolCall) => { | |
| 1355 | + if (typeof toolCall.signature === 'string') { | |
| 1356 | + addDetail(toolCall.signature, toolCall.id); | |
| 1357 | + delete toolCall.signature; | |
| 1358 | + } | |
| 1359 | + }); | |
| 1360 | + } | |
| 1361 | + if (details.length > 0) { | |
| 1362 | + message.reasoning_details = details; | |
| 1363 | + } | |
| 1364 | + } | |
| 1365 | +} | |