Merge pull request #2559 from SillyTavern/mistral-prompt-convert Improve Mistral prompt conversion

e59fb85db8e908806d631cc0432463fd91ab9a7f

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: b80ca93a1af3c49974fc2481709087f5575a2616 ef59e8abae74486de590690c5628593f03b764c6

src/endpoints/backends/chat-completions.js+1 -1 Viewed

+    }
     try {
         const messages = convertMistralMessages(request.body.messages, request.body.model, request.body.char_name, request.body.user_name);
         const controller = new AbortController();
         request.socket.removeAllListeners('close');
         request.socket.on('close', function () {

src/prompt-converters.js+13 -29 Viewed

 /**
  * Convert a prompt from the ChatML objects to the format used by MistralAI.
  * @param {object[]} messages Array of messages
 - * @param {string} model Model name
  * @param {string} charName Character name
  * @param {string} userName User name
  */
 function convertMistralMessages(messages, model, charName = '', userName = '') {
     if (!Array.isArray(messages)) {
         return [];
+    }
 -    //large seems to be throwing a 500 error if we don't make the first message a user role, most likely a bug since the other models won't do this
 +    // Make the last assistant message a prefill
 -    if (model.includes('large')) {
 -        messages[0].role = 'user';
 -    }
+-
 -    //must send a user role as last message
     const lastMsg = messages[messages.length - 1];
     if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) {
 -        if (lastMsg.role === 'assistant' && lastMsg.name) {
 +        lastMsg.prefix = true;
 -            lastMsg.content = lastMsg.name + ': ' + lastMsg.content;
 -        } else if (lastMsg.role === 'system') {
 -            lastMsg.content = '[INST] ' + lastMsg.content + ' [/INST]';
 -        }
 -        lastMsg.role = 'user';
+    }
 -    //system prompts can be stacked at the start, but any futher sys prompts after the first user/assistant message will break the model
 +    // Doesn't support completion names, so prepend if not already done by the frontend (e.g. for group chats).
 -    let encounteredNonSystemMessage = false;
     messages.forEach(msg => {
         if (msg.role === 'system' && msg.name === 'example_assistant') {
 -            if (charName) {
 +            if (charName && !msg.content.startsWith(`${charName}: `)) {
                 msg.content = `${charName}: ${msg.content}`;
+            }
             delete msg.name;
+        }
         if (msg.role === 'system' && msg.name === 'example_user') {
 -            if (userName) {
 +            if (userName && !msg.content.startsWith(`${userName}: `)) {
                 msg.content = `${userName}: ${msg.content}`;
+            }
             delete msg.name;
+        }
 -        if (msg.name) {
 +        if (msg.name && msg.role !== 'system' && !msg.content.startsWith(`${msg.name}: `)) {
             msg.content = `${msg.name}: ${msg.content}`;
             delete msg.name;
+        }
 +    });
 -        if ((msg.role === 'user' || msg.role === 'assistant') && !encounteredNonSystemMessage) {
 +    // If system role message immediately follows an assistant message, change its role to user
 -            encounteredNonSystemMessage = true;
 +    for (let i = 0; i < messages.length - 1; i++) {
 +        if (messages[i].role === 'assistant' && messages[i + 1].role === 'system') {
 +            messages[i + 1].role = 'user';
+        }
+-
 -        if (encounteredNonSystemMessage && msg.role === 'system') {
 -            msg.role = 'user';
 -            //unsure if the instruct version is what they've deployed on their endpoints and if this will make a difference or not.
 -            //it should be better than just sending the message as a user role without context though
 -            msg.content = '[INST] ' + msg.content + ' [/INST]';
+    }
 -    });
     return messages;
+}