Chat Completion: Adjust depth of 0-depth injects on continue with prefill (#4452)

ddb97d78d307640c96390e1d89ec9472cc739962

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

public/scripts/openai.js+15 -17 Viewed

     // Insert chat messages as long as there is budget available
     const chatPool = [...messages].reverse();
 -    const firstNonInjected = chatPool.find(x => !x.injected);
     for (let index = 0; index < chatPool.length; index++) {
         const chatPrompt = chatPool[index];

+        }
         if (chatCompletion.canAfford(chatMessage)) {
 -            if (type === 'continue' && oai_settings.continue_prefill && chatPrompt === firstNonInjected) {
 -                // in case we are using continue_prefill and the latest message is an assistant message, we want to prepend the users assistant prefill on the message
 -                if (chatPrompt.role === 'assistant') {
 -                    const supportsAssistantPrefill = oai_settings.chat_completion_source === chat_completion_sources.CLAUDE;
 -                    const assistantPrefill = supportsAssistantPrefill ? substituteParams(oai_settings.assistant_prefill) : '';
 -                    const messageContent = [assistantPrefill, chatMessage.content].filter(x => x).join('\n\n');
 -                    const continueMessage = await Message.createAsync(chatMessage.role, messageContent, chatMessage.identifier);
 -                    const collection = new MessageCollection('continuePrefill', continueMessage);
 -                    chatCompletion.add(collection, -1);
 -                    continue;
 -                }
 -                const collection = new MessageCollection('continuePrefill', chatMessage);
 -                chatCompletion.add(collection, -1);
 -                continue;
 -            }
+-
             chatCompletion.insertAtStart(chatMessage, 'chatHistory');
         } else {
             break;

         chatCompletion.reserveBudget(toolTokens);
+    }
 +    // Displace the message to be continued from its original position before performing in-chat injections
 +    // In case if it is an assistant message, we want to prepend the users assistant prefill on the message
 +    if (type === 'continue' && oai_settings.continue_prefill && messages.length) {
 +        const chatMessage = messages.shift();
 +        const isAssistantRole = chatMessage.role === 'assistant';
 +        const supportsAssistantPrefill = oai_settings.chat_completion_source === chat_completion_sources.CLAUDE;
 +        const namesInCompletion = oai_settings.names_behavior === character_names_behavior.COMPLETION;
 +        const assistantPrefill = isAssistantRole && supportsAssistantPrefill ? substituteParams(oai_settings.assistant_prefill) : '';
 +        const messageContent = [assistantPrefill, chatMessage.content].filter(x => x).join('\n\n');
 +        const continueMessage = await Message.createAsync(chatMessage.role, messageContent, 'continuePrefill');
 +        chatMessage.name && namesInCompletion && await continueMessage.setName(promptManager.sanitizeName(chatMessage.name));
 +        controlPrompts.add(continueMessage);
 +        chatCompletion.reserveBudget(continueMessage);
 +    }
++
     // Add in-chat injections
     messages = await populationInjectionPrompts(absolutePrompts, messages);