Merge pull request #3766 from bmen25124/custom_request_stop_string_cleanup Added stop string cleanup, better stopping string param

0a85178846cfd268a9fdd5fc4f78f7918155c3c8

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 74efb598f197e9e31b3d028984a894459af048c4 d93aba5706a084e90dee5b5e26ff85151a00d3d7
3 files changed, +95 -22Showing whitespace changes
public/scripts/custom-request.js+67 -8
@@ -2,7 +2,7 @@ import { getPresetManager } from './preset-manager.js';
22import { extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js';
33import { getTextGenServer } from './textgen-settings.js';
44import { extractReasoningFromData } from './reasoning.js';
55import { formatInstructModeChat, formatInstructModePrompt, getInstructStoppingSequences, names_behavior_types } from './instruct-mode.js';
66import { getStreamingReply, tryParseStreamingError } from './openai.js';
77import EventSourceStream from './sse-stream.js';
88
@@ -190,6 +190,7 @@ export class TextCompletionService {
190190 * @param {Object} options - Configuration options
191191 * @param {string?} [options.presetName] - Name of the preset to use for generation settings
192192 * @param {string?} [options.instructName] - Name of instruct preset for message formatting
193+ * @param {Partial<InstructSettings>?} [options.instructSettings] - Override instruct settings
193194 * @param {boolean} extractData - Whether to extract structured data from response
194195 * @param {AbortSignal?} [signal]
195196 * @returns {Promise<ExtractedData | (() => AsyncGenerator<StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
@@ -222,15 +223,20 @@ export class TextCompletionService {
222223 }
223224 }
224225
226+
227+ /** @type {InstructSettings | undefined} */
228+ let instructPreset;
225229 // Handle instruct formatting if requested
226230 if (Array.isArray(prompt) && instructName) {
227231 const instructPresetManager = getPresetManager('instruct');
228232 let instructPreset = instructPresetManager?.getCompletionPresetByName(instructName);
229233 if (instructPreset) {
230234 // Clone the preset to avoid modifying the original
231235 instructPreset = structuredClone(instructPreset);
232- instructPreset.macro = false;
233236 instructPreset.names_behavior = names_behavior_types.NONE;
237+ if (options.instructSettings) {
238+ Object.assign(instructPreset, options.instructSettings);
239+ }
234240
235241 // Format messages using instruct formatting
236242 const formattedMessages = [];
@@ -266,10 +272,9 @@ export class TextCompletionService {
266272 formattedMessages.push(messageContent);
267273 }
268274 requestData.prompt = formattedMessages.join('');
269- if (instructPreset.output_suffix) {
275+ const stoppingStrings = getInstructStoppingSequences({ customInstruct: instructPreset, useStopStrings: false });
270276 requestData.stop = [instructPreset.output_suffix]stoppingStrings;
271277 requestData.stopping_strings = [instructPreset.output_suffix]stoppingStrings;
272- }
273278 } else {
274279 console.warn(`Instruct preset "${instructName}" not found, using basic formatting`);
275280 requestData.prompt = prompt.map(x => x.content).join('\n\n');
@@ -283,7 +288,61 @@ export class TextCompletionService {
283288 // @ts-ignore
284289 const data = this.createRequestData(requestData);
285290
286291 returnconst response = await this.sendRequest(data, extractData, signal);
292+ // Remove stopping strings from the end
293+ if (!data.stream && extractData) {
294+ /** @type {ExtractedData} */
295+ // @ts-ignore
296+ const extractedData = response;
297+
298+ let message = extractedData.content;
299+
300+ message = message.replace(/[^\S\r\n]+$/gm, '');
301+
302+ if (requestData.stopping_strings) {
303+ for (const stoppingString of requestData.stopping_strings) {
304+ if (stoppingString.length) {
305+ for (let j = stoppingString.length; j > 0; j--) {
306+ if (message.slice(-j) === stoppingString.slice(0, j)) {
307+ message = message.slice(0, -j);
308+ break;
309+ }
310+ }
311+ }
312+ }
313+ }
314+
315+ if (instructPreset) {
316+ [
317+ instructPreset.stop_sequence,
318+ instructPreset.input_sequence,
319+ ].forEach(sequence => {
320+ if (sequence?.trim()) {
321+ const index = message.indexOf(sequence);
322+ if (index !== -1) {
323+ message = message.substring(0, index);
324+ }
325+ }
326+ });
327+
328+ [
329+ instructPreset.output_sequence,
330+ instructPreset.last_output_sequence,
331+ ].forEach(sequences => {
332+ if (sequences) {
333+ sequences.split('\n')
334+ .filter(line => line.trim() !== '')
335+ .forEach(line => {
336+ message = message.replaceAll(line, '');
337+ });
338+ }
339+ });
340+ }
341+
342+ extractedData.content = message;
343+ }
344+
345+ return response;
287346 }
288347
289348 /**
public/scripts/extensions/shared.js+10 -2
@@ -285,6 +285,7 @@ export class ConnectionManagerRequestService {
285285 extractData: true,
286286 includePreset: true,
287287 includeInstruct: true,
288+ instructSettings: {},
288289 };
289290
290291 static getAllowedTypes() {
@@ -298,11 +299,17 @@ export class ConnectionManagerRequestService {
298299 * @param {string} profileId
299300 * @param {string | (import('../custom-request.js').ChatCompletionMessage & {ignoreInstruct?: boolean})[]} prompt
300301 * @param {number} maxTokens
301- * @param {{stream?: boolean, signal?: AbortSignal, extractData?: boolean, includePreset?: boolean, includeInstruct?: boolean}} custom - default values are true
302+ * @param {Object} custom
303+ * @param {boolean?} [custom.stream=false]
304+ * @param {AbortSignal?} [custom.signal]
305+ * @param {boolean?} [custom.extractData=true]
306+ * @param {boolean?} [custom.includePreset=true]
307+ * @param {boolean?} [custom.includeInstruct=true]
308+ * @param {Partial<InstructSettings>?} [custom.instructSettings] Override instruct settings
302309 * @returns {Promise<import('../custom-request.js').ExtractedData | (() => AsyncGenerator<import('../custom-request.js').StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
303310 */
304311 static async sendRequest(profileId, prompt, maxTokens, custom = this.defaultSendRequestParams) {
305312 const { stream, signal, extractData, includePreset, includeInstruct, instructSettings } = { ...this.defaultSendRequestParams, ...custom };
306313
307314 const context = SillyTavern.getContext();
308315 if (context.extensionSettings.disabledExtensions.includes('connection-manager')) {
@@ -346,6 +353,7 @@ export class ConnectionManagerRequestService {
346353 }, {
347354 instructName: includeInstruct ? profile.instruct : undefined,
348355 presetName: includePreset ? profile.preset : undefined,
356+ instructSettings: includeInstruct ? instructSettings : undefined,
349357 }, extractData, signal);
350358 }
351359 default: {
public/scripts/instruct-mode.js+18 -12
@@ -243,9 +243,14 @@ export function autoSelectInstructPreset(modelId) {
243243
244244/**
245245 * Converts instruct mode sequences to an array of stopping strings.
246+ * @param {Object} options
247+ * @param {InstructSettings?} [options.customInstruct=null] - Custom instruct settings.
248+ * @param {boolean?} [options.useStopStrings] - Decides whether to use "Chat Start" and "Example Separator"
246249 * @returns {string[]} Array of instruct mode stopping strings.
247250 */
248-export function getInstructStoppingSequences() {
251+export function getInstructStoppingSequences({ customInstruct = null, useStopStrings = null } = {}) {
252+ const instruct = structuredClone(customInstruct ?? power_user.instruct);
253+
249254 /**
250255 * Adds instruct mode sequence to the result array.
251256 * @param {string} sequence Sequence string.
@@ -254,7 +259,7 @@ export function getInstructStoppingSequences() {
254259 function addInstructSequence(sequence) {
255260 // Cohee: oobabooga's textgen always appends newline before the sequence as a stopping string
256261 // But it's a problem for Metharme which doesn't use newlines to separate them.
257262 const wrap = (s) => power_user.instruct.wrap ? '\n' + s : s;
258263 // Sequence must be a non-empty string
259264 if (typeof sequence === 'string' && sequence.length > 0) {
260265 // If sequence is just a whitespace or newline - we don't want to make it a stopping string
@@ -262,7 +267,7 @@ export function getInstructStoppingSequences() {
262267 if (sequence.trim().length > 0) {
263268 const wrappedSequence = wrap(sequence);
264269 // Need to respect "insert macro" setting
265270 const stopString = power_user.instruct.macro ? substituteParams(wrappedSequence) : wrappedSequence;
266271 result.push(stopString);
267272 }
268273 }
@@ -270,14 +275,15 @@ export function getInstructStoppingSequences() {
270275
271276 const result = [];
272277
273- if (power_user.instruct.enabled) {
278+ // Since preset's don't have "enabled", we assume it's always enabled
274- const stop_sequence = power_user.instruct.stop_sequence || '';
279+ if (customInstruct ?? instruct.enabled) {
275280 const input_sequencestop_sequence = power_user.instruct.input_sequence?.replace(/{{name}}/gi, name1)stop_sequence || '';
276281 const output_sequenceinput_sequence = power_user.instruct.output_sequenceinput_sequence?.replace(/{{name}}/gi, name2name1) || '';
277282 const first_output_sequenceoutput_sequence = power_user.instruct.first_output_sequenceoutput_sequence?.replace(/{{name}}/gi, name2) || '';
278283 const last_output_sequencefirst_output_sequence = power_user.instruct.last_output_sequencefirst_output_sequence?.replace(/{{name}}/gi, name2) || '';
279284 const system_sequencelast_output_sequence = power_user.instruct.system_sequencelast_output_sequence?.replace(/{{name}}/gi, 'System'name2) || '';
280285 const last_system_sequencesystem_sequence = power_user.instruct.last_system_sequencesystem_sequence?.replace(/{{name}}/gi, 'System') || '';
286+ const last_system_sequence = instruct.last_system_sequence?.replace(/{{name}}/gi, 'System') || '';
281287
282288 const combined_sequence = [
283289 stop_sequence,
@@ -292,7 +298,7 @@ export function getInstructStoppingSequences() {
292298 combined_sequence.split('\n').filter((line, index, self) => self.indexOf(line) === index).forEach(addInstructSequence);
293299 }
294300
295301 if (useStopStrings ?? power_user.context.use_stop_strings) {
296302 if (power_user.context.chat_start) {
297303 result.push(`\n${substituteParams(power_user.context.chat_start)}`);
298304 }