1
import {
2
    abortStatusCheck,
3
    eventSource,
4
    event_types,
5
    getRequestHeaders,
6
    getStoppingStrings,
7
    main_api,
8
    max_context,
9
    online_status,
10
    resultCheckStatus,
11
    saveSettingsDebounced,
12
    setGenerationParamsFromPreset,
13
    setOnlineStatus,
14
    startStatusLoading,
15
    substituteParams,
16
} from '../script.js';
17
import { deriveTemplatesFromChatTemplate } from './chat-templates.js';
18
import { t } from './i18n.js';
19
import { autoSelectInstructPreset, selectContextPreset, selectInstructPreset } from './instruct-mode.js';
20
import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasListResult } from './logit-bias.js';
21

22
import { power_user, registerDebugFunction } from './power-user.js';
23
import { getActiveManualApiSamplers, loadApiSelectedSamplers, isSamplerManualPriorityEnabled } from './samplerSelect.js';
24
import { SECRET_KEYS, writeSecret } from './secrets.js';
25
import { getEventSourceStream } from './sse-stream.js';
26
import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, loadAphroditeModels, loadDreamGenModels, loadFeatherlessModels, loadGenericModels, loadInfermaticAIModels, loadLlamaCppModels, loadMancerModels, loadOllamaModels, loadOpenRouterModels, loadTabbyModels, loadTogetherAIModels, loadVllmModels, updateOpenRouterProvidersWarning } from './textgen-models.js';
27
import { ENCODE_TOKENIZERS, TEXTGEN_TOKENIZERS, TOKENIZER_SUPPORTED_KEY, getTextTokens, getTokenizerBestMatch, tokenizers } from './tokenizers.js';
28
import { AbortReason } from './util/AbortReason.js';
29
import { getSortableDelay, onlyUnique, arraysEqual, isObject } from './utils.js';
30

31
export const textgen_types = {
32
    OOBA: 'ooba',
33
    MANCER: 'mancer',
34
    VLLM: 'vllm',
35
    APHRODITE: 'aphrodite',
36
    TABBY: 'tabby',
37
    KOBOLDCPP: 'koboldcpp',
38
    TOGETHERAI: 'togetherai',
39
    LLAMACPP: 'llamacpp',
40
    OLLAMA: 'ollama',
41
    INFERMATICAI: 'infermaticai',
42
    DREAMGEN: 'dreamgen',
43
    OPENROUTER: 'openrouter',
44
    FEATHERLESS: 'featherless',
45
    HUGGINGFACE: 'huggingface',
46
    GENERIC: 'generic',
47
};
48

49
const {
50
    GENERIC,
51
    MANCER,
52
    VLLM,
53
    APHRODITE,
54
    TABBY,
55
    TOGETHERAI,
56
    OOBA,
57
    OLLAMA,
58
    LLAMACPP,
59
    INFERMATICAI,
60
    DREAMGEN,
61
    OPENROUTER,
62
    KOBOLDCPP,
63
    HUGGINGFACE,
64
    FEATHERLESS,
65
} = textgen_types;
66

67
const LLAMACPP_DEFAULT_ORDER = [
68
    'penalties',
69
    'dry',
70
    'top_n_sigma',
71
    'top_k',
72
    'typ_p',
73
    'top_p',
74
    'min_p',
75
    'xtc',
76
    'temperature',
77
    'adaptive_p',
78
];
79
const OOBA_DEFAULT_ORDER = [
80
    'repetition_penalty',
81
    'presence_penalty',
82
    'frequency_penalty',
83
    'dry',
84
    'temperature',
85
    'dynamic_temperature',
86
    'quadratic_sampling',
87
    'top_n_sigma',
88
    'top_k',
89
    'top_p',
90
    'typical_p',
91
    'epsilon_cutoff',
92
    'eta_cutoff',
93
    'tfs',
94
    'top_a',
95
    'min_p',
96
    'adaptive_p',
97
    'mirostat',
98
    'xtc',
99
    'encoder_repetition_penalty',
100
    'no_repeat_ngram',
101
];
102
export const APHRODITE_DEFAULT_ORDER = [
103
    'dry',
104
    'penalties',
105
    'no_repeat_ngram',
106
    'temperature',
107
    'top_nsigma',
108
    'top_p_top_k',
109
    'top_a',
110
    'min_p',
111
    'tfs',
112
    'eta_cutoff',
113
    'epsilon_cutoff',
114
    'typical_p',
115
    'quadratic',
116
    'xtc',
117
];
118
const BIAS_KEY = '#textgenerationwebui_api-settings';
119

120
// Maybe let it be configurable in the future?
121
// (7 days later) The future has come.
122
const MANCER_SERVER_KEY = 'mancer_server';
123
const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech';
124
export let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
125
export let TOGETHERAI_SERVER = 'https://api.together.xyz';
126
export let INFERMATICAI_SERVER = 'https://api.totalgpt.ai';
127
export let DREAMGEN_SERVER = 'https://dreamgen.com';
128
export let OPENROUTER_SERVER = 'https://openrouter.ai/api';
129
export let FEATHERLESS_SERVER = 'https://api.featherless.ai/v1';
130

131
export const SERVER_INPUTS = {
132
    [textgen_types.OOBA]: '#textgenerationwebui_api_url_text',
133
    [textgen_types.VLLM]: '#vllm_api_url_text',
134
    [textgen_types.APHRODITE]: '#aphrodite_api_url_text',
135
    [textgen_types.TABBY]: '#tabby_api_url_text',
136
    [textgen_types.KOBOLDCPP]: '#koboldcpp_api_url_text',
137
    [textgen_types.LLAMACPP]: '#llamacpp_api_url_text',
138
    [textgen_types.OLLAMA]: '#ollama_api_url_text',
139
    [textgen_types.HUGGINGFACE]: '#huggingface_api_url_text',
140
    [textgen_types.GENERIC]: '#generic_api_url_text',
141
};
142

143
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
144
export const textgenerationwebui_settings = {
145
    temp: 0.7,
146
    temperature_last: true,
147
    top_p: 0.5,
148
    top_k: 40,
149
    top_a: 0,
150
    tfs: 1,
151
    epsilon_cutoff: 0,
152
    eta_cutoff: 0,
153
    typical_p: 1,
154
    min_p: 0,
155
    rep_pen: 1.2,
156
    rep_pen_range: 0,
157
    rep_pen_decay: 0,
158
    rep_pen_slope: 1,
159
    no_repeat_ngram_size: 0,
160
    penalty_alpha: 0,
161
    num_beams: 1,
162
    length_penalty: 1,
163
    min_length: 0,
164
    encoder_rep_pen: 1,
165
    freq_pen: 0,
166
    presence_pen: 0,
167
    skew: 0,
168
    do_sample: true,
169
    early_stopping: false,
170
    dynatemp: false,
171
    min_temp: 0,
172
    max_temp: 2.0,
173
    dynatemp_exponent: 1.0,
174
    smoothing_factor: 0.0,
175
    smoothing_curve: 1.0,
176
    dry_allowed_length: 2,
177
    dry_multiplier: 0.0,
178
    dry_base: 1.75,
179
    dry_sequence_breakers: '["\\n", ":", "\\"", "*"]',
180
    dry_penalty_last_n: 0,
181
    max_tokens_second: 0,
182
    seed: -1,
183
    preset: 'Default',
184
    add_bos_token: true,
185
    stopping_strings: [],
186
    //truncation_length: 2048,
187
    ban_eos_token: false,
188
    skip_special_tokens: true,
189
    include_reasoning: true,
190
    streaming: false,
191
    mirostat_mode: 0,
192
    mirostat_tau: 5,
193
    mirostat_eta: 0.1,
194
    guidance_scale: 1,
195
    negative_prompt: '',
196
    grammar_string: '',
197
    json_schema: null,
198
    json_schema_allow_empty: false,
199
    banned_tokens: '',
200
    global_banned_tokens: '',
201
    send_banned_tokens: true,
202
    sampler_priority: OOBA_DEFAULT_ORDER,
203
    samplers: LLAMACPP_DEFAULT_ORDER,
204
    samplers_priorities: APHRODITE_DEFAULT_ORDER,
205
    ignore_eos_token: false,
206
    spaces_between_special_tokens: true,
207
    speculative_ngram: false,
208
    type: textgen_types.OOBA,
209
    mancer_model: 'mytholite',
210
    togetherai_model: 'Gryphe/MythoMax-L2-13b',
211
    infermaticai_model: '',
212
    ollama_model: '',
213
    openrouter_model: 'openrouter/auto',
214
    openrouter_providers: [],
215
    openrouter_quantizations: [],
216
    vllm_model: '',
217
    aphrodite_model: '',
218
    dreamgen_model: 'lucid-v1-extra-large/text',
219
    tabby_model: '',
220
    llamacpp_model: '',
221
    sampler_order: KOBOLDCPP_ORDER,
222
    logit_bias: [],
223
    n: 1,
224
    server_urls: {},
225
    custom_model: '',
226
    bypass_status_check: false,
227
    openrouter_allow_fallbacks: true,
228
    xtc_threshold: 0.1,
229
    xtc_probability: 0,
230
    nsigma: 0.0,
231
    min_keep: 0,
232
    featherless_model: '',
233
    generic_model: '',
234
    extensions: {},
235
    adaptive_target: -0.01,
236
    adaptive_decay: 0.9,
237
};
238

239
export {
240
    showSamplerControls as showTGSamplerControls,
241
};
242

243
export let textgenerationwebui_banned_in_macros = [];
244

245
export let textgenerationwebui_presets = [];
246
export let textgenerationwebui_preset_names = [];
247

248
export const setting_names = [
249
    'temp',
250
    'temperature_last',
251
    'rep_pen',
252
    'rep_pen_range',
253
    'rep_pen_decay',
254
    'rep_pen_slope',
255
    'no_repeat_ngram_size',
256
    'top_k',
257
    'top_p',
258
    'top_a',
259
    'tfs',
260
    'epsilon_cutoff',
261
    'eta_cutoff',
262
    'typical_p',
263
    'min_p',
264
    'penalty_alpha',
265
    'num_beams',
266
    'length_penalty',
267
    'min_length',
268
    'dynatemp',
269
    'min_temp',
270
    'max_temp',
271
    'dynatemp_exponent',
272
    'smoothing_factor',
273
    'smoothing_curve',
274
    'dry_allowed_length',
275
    'dry_multiplier',
276
    'dry_base',
277
    'dry_sequence_breakers',
278
    'dry_penalty_last_n',
279
    'max_tokens_second',
280
    'encoder_rep_pen',
281
    'freq_pen',
282
    'presence_pen',
283
    'skew',
284
    'do_sample',
285
    'early_stopping',
286
    'seed',
287
    'add_bos_token',
288
    'ban_eos_token',
289
    'skip_special_tokens',
290
    'include_reasoning',
291
    'streaming',
292
    'mirostat_mode',
293
    'mirostat_tau',
294
    'mirostat_eta',
295
    'guidance_scale',
296
    'negative_prompt',
297
    'grammar_string',
298
    'json_schema',
299
    'banned_tokens',
300
    'global_banned_tokens',
301
    'send_banned_tokens',
302
    'ignore_eos_token',
303
    'spaces_between_special_tokens',
304
    'speculative_ngram',
305
    'sampler_order',
306
    'sampler_priority',
307
    'samplers',
308
    'samplers_priorities',
309
    'n',
310
    'logit_bias',
311
    'custom_model',
312
    'bypass_status_check',
313
    'openrouter_allow_fallbacks',
314
    'xtc_threshold',
315
    'xtc_probability',
316
    'nsigma',
317
    'min_keep',
318
    'generic_model',
319
    'extensions',
320
    'json_schema_allow_empty',
321
    'adaptive_target',
322
    'adaptive_decay',
323
];
324

325
const DYNATEMP_BLOCK = document.getElementById('dynatemp_block_ooba');
326

327
export function validateTextGenUrl() {
328
    const selector = SERVER_INPUTS[textgenerationwebui_settings.type];
329

330
    if (!selector) {
331
        return;
332
    }
333

334
    const control = $(selector);
335
    const url = String(control.val()).trim();
336
    const formattedUrl = formatTextGenURL(url);
337

338
    if (!formattedUrl) {
339
        toastr.error(t`Enter a valid API URL`, 'Text Completion API');
340
        return;
341
    }
342

343
    control.val(formattedUrl);
344
}
345

346
/**
347
 * Gets the API URL for the selected text generation type.
348
 * @param {string} type If it's set, ignores active type
349
 * @returns {string} API URL
350
 */
351
export function getTextGenServer(type = null) {
352
    const selectedType = type ?? textgenerationwebui_settings.type;
353
    switch (selectedType) {
354
        case FEATHERLESS:
355
            return FEATHERLESS_SERVER;
356
        case MANCER:
357
            return MANCER_SERVER;
358
        case TOGETHERAI:
359
            return TOGETHERAI_SERVER;
360
        case INFERMATICAI:
361
            return INFERMATICAI_SERVER;
362
        case DREAMGEN:
363
            return DREAMGEN_SERVER;
364
        case OPENROUTER:
365
            return OPENROUTER_SERVER;
366
        default:
367
            return textgenerationwebui_settings.server_urls[selectedType] ?? '';
368
    }
369
}
370

371
async function selectPreset(name) {
372
    const preset = textgenerationwebui_presets[textgenerationwebui_preset_names.indexOf(name)];
373

374
    if (!preset) {
375
        return;
376
    }
377

378
    textgenerationwebui_settings.preset = name;
379
    for (const name of setting_names) {
380
        const value = preset[name];
381
        setSettingByName(name, value, true);
382
    }
383
    setGenerationParamsFromPreset(preset);
384
    BIAS_CACHE.delete(BIAS_KEY);
385
    displayLogitBias(preset.logit_bias, BIAS_KEY);
386
    saveSettingsDebounced();
387
}
388

389
export function formatTextGenURL(value) {
390
    try {
391
        const noFormatTypes = [MANCER, TOGETHERAI, INFERMATICAI, DREAMGEN, OPENROUTER];
392
        if (noFormatTypes.includes(textgenerationwebui_settings.type)) {
393
            return value;
394
        }
395

396
        const url = new URL(value);
397
        return url.toString();
398
    } catch {
399
        // Just using URL as a validation check
400
    }
401
    return null;
402
}
403

404
function convertPresets(presets) {
405
    return Array.isArray(presets) ? presets.map((p) => JSON.parse(p)) : [];
406
}
407

408
function getTokenizerForTokenIds() {
409
    const bestMatchTokenizer = getTokenizerBestMatch('textgenerationwebui');
410
    if (bestMatchTokenizer === tokenizers.API_TEXTGENERATIONWEBUI) {
411
        return tokenizers.API_CURRENT;
412
    }
413

414
    if (power_user.tokenizer === tokenizers.API_CURRENT && TEXTGEN_TOKENIZERS.includes(textgenerationwebui_settings.type)) {
415
        return tokenizers.API_CURRENT;
416
    }
417

418
    if (ENCODE_TOKENIZERS.includes(power_user.tokenizer)) {
419
        return power_user.tokenizer;
420
    }
421

422
    if (textgenerationwebui_settings.type === OPENROUTER) {
423
        return getCurrentOpenRouterModelTokenizer();
424
    }
425

426
    if (textgenerationwebui_settings.type === DREAMGEN) {
427
        return getCurrentDreamGenModelTokenizer();
428
    }
429

430
    return tokenizers.LLAMA;
431
}
432

433
/**
434
 * Gets the custom token bans from settings and macros.
435
 * @param {TextCompletionSettings} settings Text completion settings to use
436
 * @typedef {{banned_tokens: string, banned_strings: string[]}} TokenBanResult
437
 * @returns {TokenBanResult} String with comma-separated banned token IDs
438
 */
439
function getCustomTokenBans(settings = null) {
440
    settings = settings ?? textgenerationwebui_settings;
441
    if (!settings.send_banned_tokens || (!settings.banned_tokens && !settings.global_banned_tokens && !textgenerationwebui_banned_in_macros.length)) {
442
        return {
443
            banned_tokens: '',
444
            banned_strings: [],
445
        };
446
    }
447

448
    const tokenizer = getTokenizerForTokenIds();
449
    const banned_tokens = [];
450
    const banned_strings = [];
451
    const sequences = []
452
        .concat(settings.banned_tokens.split('\n'))
453
        .concat(settings.global_banned_tokens.split('\n'))
454
        .concat(textgenerationwebui_banned_in_macros)
455
        .filter(x => x.length > 0)
456
        .filter(onlyUnique)
457
        .map(x => substituteParams(x));
458

459
    //debug
460
    if (textgenerationwebui_banned_in_macros.length) {
461
        console.log('=== Found banned word sequences in the macros:', textgenerationwebui_banned_in_macros, 'Resulting array of banned sequences (will be used this generation turn):', sequences);
462
    }
463

464
    //clean old temporary bans found in macros before, for the next generation turn.
465
    textgenerationwebui_banned_in_macros = [];
466

467
    for (const line of sequences) {
468
        // Raw token ids, JSON serialized
469
        if (line.startsWith('[') && line.endsWith(']')) {
470
            try {
471
                const tokens = JSON.parse(line);
472

473
                if (Array.isArray(tokens) && tokens.every(t => Number.isInteger(t))) {
474
                    banned_tokens.push(...tokens);
475
                } else {
476
                    throw new Error('Not an array of integers');
477
                }
478
            } catch (err) {
479
                console.log(`Failed to parse bad word token list: ${line}`, err);
480
            }
481
        } else if (line.startsWith('"') && line.endsWith('"')) {
482
            // Remove the enclosing quotes
483

484
            banned_strings.push(line.slice(1, -1));
485
        } else {
486
            try {
487
                const tokens = getTextTokens(tokenizer, line);
488
                banned_tokens.push(...tokens);
489
            } catch {
490
                console.log(`Could not tokenize raw text: ${line}`);
491
            }
492
        }
493
    }
494

495
    return {
496
        banned_tokens: banned_tokens.filter(onlyUnique).map(x => String(x)).join(','),
497
        banned_strings: banned_strings,
498
    };
499
}
500

501
/**
502
 * Sets the banned strings kill switch toggle.
503
 * @param {boolean} isEnabled Kill switch state
504
 * @param {string} title Label title
505
 */
506
function toggleBannedStringsKillSwitch(isEnabled, title) {
507
    $('#send_banned_tokens_textgenerationwebui').prop('checked', isEnabled);
508
    $('#send_banned_tokens_label').find('.menu_button').toggleClass('toggleEnabled', isEnabled).prop('title', title);
509
    textgenerationwebui_settings.send_banned_tokens = isEnabled;
510
    saveSettingsDebounced();
511
}
512

513
/**
514
 * Calculates logit bias object from the logit bias list.
515
 * @param {TextCompletionSettings} settings Text completion settings
516
 * @returns {object} Logit bias object
517
 */
518
function calculateLogitBias(settings = null) {
519
    settings = settings ?? textgenerationwebui_settings;
520

521
    if (!Array.isArray(settings.logit_bias) || settings.logit_bias.length === 0) {
522
        return {};
523
    }
524

525
    const tokenizer = getTokenizerForTokenIds();
526
    const result = {};
527

528
    /**
529
     * Adds bias to the logit bias object.
530
     * @param {number} bias
531
     * @param {number[]} sequence
532
     * @returns {object} Accumulated logit bias object
533
     */
534
    function addBias(bias, sequence) {
535
        if (sequence.length === 0) {
536
            return;
537
        }
538

539
        for (const logit of sequence) {
540
            const key = String(logit);
541
            result[key] = bias;
542
        }
543

544
        return result;
545
    }
546

547
    getLogitBiasListResult(settings.logit_bias, tokenizer, addBias);
548

549
    return result;
550
}
551

552
export async function loadTextGenSettings(data, loadedSettings) {
553
    await loadApiSelectedSamplers();
554
    textgenerationwebui_presets = convertPresets(data.textgenerationwebui_presets);
555
    textgenerationwebui_preset_names = data.textgenerationwebui_preset_names ?? [];
556
    Object.assign(textgenerationwebui_settings, loadedSettings.textgenerationwebui_settings ?? {});
557

558
    if (loadedSettings.api_server_textgenerationwebui) {
559
        for (const type of Object.keys(SERVER_INPUTS)) {
560
            textgenerationwebui_settings.server_urls[type] = loadedSettings.api_server_textgenerationwebui;
561
        }
562
        delete loadedSettings.api_server_textgenerationwebui;
563
    }
564

565
    for (const [type, selector] of Object.entries(SERVER_INPUTS)) {
566
        const control = $(selector);
567
        control.val(textgenerationwebui_settings.server_urls[type] ?? '').on('input', function () {
568
            textgenerationwebui_settings.server_urls[type] = String($(this).val()).trim();
569
            saveSettingsDebounced();
570
        });
571
    }
572

573
    if (loadedSettings.api_use_mancer_webui) {
574
        textgenerationwebui_settings.type = MANCER;
575
    }
576

577
    for (const name of textgenerationwebui_preset_names) {
578
        const option = document.createElement('option');
579
        option.value = name;
580
        option.innerText = name;
581
        $('#settings_preset_textgenerationwebui').append(option);
582
    }
583

584
    if (textgenerationwebui_settings.preset) {
585
        $('#settings_preset_textgenerationwebui').val(textgenerationwebui_settings.preset);
586
    }
587

588
    for (const i of setting_names) {
589
        const value = textgenerationwebui_settings[i];
590
        setSettingByName(i, value);
591
    }
592

593
    $('#textgen_type').val(textgenerationwebui_settings.type);
594
    $('#openrouter_providers_text').val(textgenerationwebui_settings.openrouter_providers).trigger('change');
595
    $('#openrouter_quantizations_text').val(textgenerationwebui_settings.openrouter_quantizations).trigger('change');
596
    showSamplerControls(textgenerationwebui_settings.type);
597
    BIAS_CACHE.delete(BIAS_KEY);
598
    displayLogitBias(textgenerationwebui_settings.logit_bias, BIAS_KEY);
599

600
    registerDebugFunction('change-mancer-url', 'Change Mancer base URL', 'Change Mancer API server base URL', () => {
601
        const result = prompt(`Enter Mancer base URL\nDefault: ${MANCER_SERVER_DEFAULT}`, MANCER_SERVER);
602

603
        if (result) {
604
            localStorage.setItem(MANCER_SERVER_KEY, result);
605
            MANCER_SERVER = result;
606
        }
607
    });
608
}
609

610
/**
611
 * Sorts the sampler items by the given order.
612
 * @param {any[]} orderArray Sampler order array.
613
 */
614
function sortKoboldItemsByOrder(orderArray) {
615
    console.debug('Preset samplers order: ' + orderArray);
616
    const $draggableItems = $('#koboldcpp_order');
617

618
    for (let i = 0; i < orderArray.length; i++) {
619
        const index = orderArray[i];
620
        const $item = $draggableItems.find(`[data-id="${index}"]`).detach();
621
        $draggableItems.append($item);
622
    }
623
}
624

625
function sortLlamacppItemsByOrder(orderArray) {
626
    console.debug('Preset samplers order: ', orderArray);
627
    const $container = $('#llamacpp_samplers_sortable');
628

629
    orderArray.forEach((name) => {
630
        const $item = $container.find(`[data-name="${name}"]`).detach();
631
        $container.append($item);
632
    });
633
}
634

635
function sortOobaItemsByOrder(orderArray) {
636
    console.debug('Preset samplers order: ', orderArray);
637
    const $container = $('#sampler_priority_container');
638

639
    orderArray.forEach((name) => {
640
        const $item = $container.find(`[data-name="${name}"]`).detach();
641
        $container.append($item);
642
    });
643
}
644

645
/**
646
 * Sorts the Aphrodite sampler items by the given order.
647
 * @param {string[]} orderArray Sampler order array.
648
 */
649
function sortAphroditeItemsByOrder(orderArray) {
650
    console.debug('Preset samplers order: ', orderArray);
651
    const $container = $('#sampler_priority_container_aphrodite');
652

653
    orderArray.forEach((name) => {
654
        const $item = $container.find(`[data-name="${name}"]`).detach();
655
        $container.append($item);
656
    });
657
}
658

659
async function getStatusTextgen() {
660
    const url = '/api/backends/text-completions/status';
661

662
    const endpoint = getTextGenServer();
663

664
    if (!endpoint) {
665
        console.warn('No endpoint for status check');
666
        setOnlineStatus('no_connection');
667
        return resultCheckStatus();
668
    }
669

670
    // Clear logit bias cache
671
    BIAS_CACHE.delete(BIAS_KEY);
672

673
    if ([textgen_types.GENERIC, textgen_types.OOBA].includes(textgenerationwebui_settings.type) && textgenerationwebui_settings.bypass_status_check) {
674
        setOnlineStatus(t`Status check bypassed`);
675
        return resultCheckStatus();
676
    }
677

678
    try {
679
        const response = await fetch(url, {
680
            method: 'POST',
681
            headers: getRequestHeaders(),
682
            body: JSON.stringify({
683
                api_server: endpoint,
684
                api_type: textgenerationwebui_settings.type,
685
            }),
686
            signal: abortStatusCheck.signal,
687
        });
688

689
        const data = await response.json();
690

691
        if (textgenerationwebui_settings.type === textgen_types.MANCER) {
692
            loadMancerModels(data?.data);
693
            setOnlineStatus(textgenerationwebui_settings.mancer_model);
694
        } else if (textgenerationwebui_settings.type === textgen_types.TOGETHERAI) {
695
            loadTogetherAIModels(data?.data);
696
            setOnlineStatus(textgenerationwebui_settings.togetherai_model);
697
        } else if (textgenerationwebui_settings.type === textgen_types.OLLAMA) {
698
            loadOllamaModels(data?.data);
699
            setOnlineStatus(textgenerationwebui_settings.ollama_model || t`Connected`);
700
        } else if (textgenerationwebui_settings.type === textgen_types.INFERMATICAI) {
701
            loadInfermaticAIModels(data?.data);
702
            setOnlineStatus(textgenerationwebui_settings.infermaticai_model);
703
        } else if (textgenerationwebui_settings.type === textgen_types.DREAMGEN) {
704
            loadDreamGenModels(data?.data);
705
            setOnlineStatus(textgenerationwebui_settings.dreamgen_model);
706
        } else if (textgenerationwebui_settings.type === textgen_types.OPENROUTER) {
707
            loadOpenRouterModels(data?.data);
708
            setOnlineStatus(textgenerationwebui_settings.openrouter_model);
709
        } else if (textgenerationwebui_settings.type === textgen_types.VLLM) {
710
            loadVllmModels(data?.data);
711
            setOnlineStatus(textgenerationwebui_settings.vllm_model);
712
        } else if (textgenerationwebui_settings.type === textgen_types.APHRODITE) {
713
            loadAphroditeModels(data?.data);
714
            setOnlineStatus(textgenerationwebui_settings.aphrodite_model);
715
        } else if (textgenerationwebui_settings.type === textgen_types.FEATHERLESS) {
716
            loadFeatherlessModels(data?.data);
717
            setOnlineStatus(textgenerationwebui_settings.featherless_model);
718
        } else if (textgenerationwebui_settings.type === textgen_types.TABBY) {
719
            loadTabbyModels(data?.data);
720
            setOnlineStatus(textgenerationwebui_settings.tabby_model || data?.result);
721
        } else if (textgenerationwebui_settings.type === textgen_types.LLAMACPP) {
722
            loadLlamaCppModels(data?.data);
723
            setOnlineStatus(textgenerationwebui_settings.llamacpp_model || data?.result || t`Connected`);
724
        } else if (textgenerationwebui_settings.type === textgen_types.GENERIC) {
725
            loadGenericModels(data?.data);
726
            setOnlineStatus(textgenerationwebui_settings.generic_model || data?.result || t`Connected`);
727
        } else {
728
            setOnlineStatus(data?.result);
729
        }
730

731
        if (!online_status) {
732
            setOnlineStatus('no_connection');
733
        }
734

735
        power_user.chat_template_hash = '';
736

737
        // Determine instruct mode preset
738
        const autoSelected = autoSelectInstructPreset(online_status);
739

740
        const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
741
        supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
742

743
        const wantsInstructDerivation = !autoSelected && (power_user.instruct.enabled && power_user.instruct_derived);
744
        const wantsContextDerivation = !autoSelected && power_user.context_derived;
745
        const wantsContextSize = power_user.context_size_derived;
746
        const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgenerationwebui_settings.type);
747

748
        if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation || wantsContextSize)) {
749
            const model = textgenerationwebui_settings.type === textgen_types.LLAMACPP
750
                ? textgenerationwebui_settings.llamacpp_model
751
                : undefined;
752

753
            const response = await fetch('/api/backends/text-completions/props', {
754
                method: 'POST',
755
                headers: getRequestHeaders(),
756
                body: JSON.stringify({
757
                    api_server: endpoint,
758
                    api_type: textgenerationwebui_settings.type,
759
                    model: model,
760
                }),
761
            });
762

763
            if (response.ok) {
764
                const data = await response.json();
765
                if (data) {
766
                    const { chat_template, chat_template_hash } = data;
767
                    power_user.chat_template_hash = chat_template_hash;
768

769
                    if (wantsContextSize && 'default_generation_settings' in data) {
770
                        const backend_max_context = data.default_generation_settings.n_ctx;
771
                        if (backend_max_context && typeof backend_max_context === 'number') {
772
                            const old_value = max_context;
773
                            if (max_context !== backend_max_context) {
774
                                setGenerationParamsFromPreset({ max_length: backend_max_context });
775
                            }
776
                            if (old_value !== max_context) {
777
                                console.log(`Auto-switched max context from ${old_value} to ${max_context}`);
778
                                toastr.info(`${old_value} ⇒ ${max_context}`, 'Context Size Changed');
779
                            }
780
                        }
781
                    }
782
                    console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
783
                    const savedTemplate = power_user.model_templates_mappings[chat_template_hash];
784
                    const derivedTemplate = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
785
                    const { context, instruct } = savedTemplate ?? derivedTemplate;
786

787
                    if (wantsContextDerivation && context) {
788
                        selectContextPreset(context, { isAuto: true });
789
                    }
790
                    if (wantsInstructDerivation && power_user.instruct.enabled && instruct) {
791
                        selectInstructPreset(instruct, { isAuto: true });
792
                    }
793
                }
794
            }
795
        }
796

797
        // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
798
        if (online_status === 'no_connection' && data.response) {
799
            toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });
800
        }
801
    } catch (err) {
802
        if (err instanceof AbortReason) {
803
            console.info('Status check aborted.', err.reason);
804
        } else {
805
            console.error('Error getting status', err);
806
        }
807
        setOnlineStatus('no_connection');
808
    }
809

810
    return resultCheckStatus();
811
}
812

813
export function initTextGenSettings() {
814
    $('#send_banned_tokens_textgenerationwebui').on('change', function () {
815
        const checked = !!$(this).prop('checked');
816
        toggleBannedStringsKillSwitch(checked,
817
            checked
818
                ? t`Banned tokens/strings are being sent in the request.`
819
                : t`Banned tokens/strings are NOT being sent in the request.`);
820
    });
821

822
    $('#koboldcpp_order').sortable({
823
        delay: getSortableDelay(),
824
        stop: function () {
825
            const order = [];
826
            $('#koboldcpp_order').children().each(function () {
827
                order.push($(this).data('id'));
828
            });
829
            textgenerationwebui_settings.sampler_order = order;
830
            console.log('Samplers reordered:', textgenerationwebui_settings.sampler_order);
831
            saveSettingsDebounced();
832
        },
833
    });
834

835
    $('#koboldcpp_default_order').on('click', function () {
836
        textgenerationwebui_settings.sampler_order = KOBOLDCPP_ORDER;
837
        sortKoboldItemsByOrder(textgenerationwebui_settings.sampler_order);
838
        saveSettingsDebounced();
839
    });
840

841
    $('#llamacpp_samplers_sortable').sortable({
842
        delay: getSortableDelay(),
843
        stop: function () {
844
            const order = [];
845
            $('#llamacpp_samplers_sortable').children().each(function () {
846
                order.push($(this).data('name'));
847
            });
848
            textgenerationwebui_settings.samplers = order;
849
            console.log('Samplers reordered:', textgenerationwebui_settings.samplers);
850
            saveSettingsDebounced();
851
        },
852
    });
853

854
    $('#llamacpp_samplers_default_order').on('click', function () {
855
        sortLlamacppItemsByOrder(LLAMACPP_DEFAULT_ORDER);
856
        textgenerationwebui_settings.samplers = LLAMACPP_DEFAULT_ORDER;
857
        console.log('Default samplers order loaded:', textgenerationwebui_settings.samplers);
858
        saveSettingsDebounced();
859
    });
860

861
    $('#sampler_priority_container').sortable({
862
        delay: getSortableDelay(),
863
        stop: function () {
864
            const order = [];
865
            $('#sampler_priority_container').children().each(function () {
866
                order.push($(this).data('name'));
867
            });
868
            textgenerationwebui_settings.sampler_priority = order;
869
            console.log('Samplers reordered:', textgenerationwebui_settings.sampler_priority);
870
            saveSettingsDebounced();
871
        },
872
    });
873

874
    $('#sampler_priority_container_aphrodite').sortable({
875
        delay: getSortableDelay(),
876
        stop: function () {
877
            const order = [];
878
            $('#sampler_priority_container_aphrodite').children().each(function () {
879
                order.push($(this).data('name'));
880
            });
881
            textgenerationwebui_settings.samplers_priorities = order;
882
            console.log('Samplers reordered:', textgenerationwebui_settings.samplers_priorities);
883
            saveSettingsDebounced();
884
        },
885
    });
886

887
    $('#tabby_json_schema').on('input', function () {
888
        const json_schema_string = String($(this).val());
889

890
        if (json_schema_string) {
891
            try {
892
                textgenerationwebui_settings.json_schema = JSON.parse(json_schema_string);
893
            } catch {
894
                textgenerationwebui_settings.json_schema = null;
895
            }
896
        } else {
897
            textgenerationwebui_settings.json_schema = null;
898
        }
899

900
        saveSettingsDebounced();
901
    });
902

903
    $('#textgenerationwebui_default_order').on('click', function () {
904
        sortOobaItemsByOrder(OOBA_DEFAULT_ORDER);
905
        textgenerationwebui_settings.sampler_priority = OOBA_DEFAULT_ORDER;
906
        console.log('Default samplers order loaded:', textgenerationwebui_settings.sampler_priority);
907
        saveSettingsDebounced();
908
    });
909

910
    $('#aphrodite_default_order').on('click', function () {
911
        sortAphroditeItemsByOrder(APHRODITE_DEFAULT_ORDER);
912
        textgenerationwebui_settings.samplers_priorities = APHRODITE_DEFAULT_ORDER;
913
        console.log('Default samplers order loaded:', textgenerationwebui_settings.samplers_priorities);
914
        saveSettingsDebounced();
915
    });
916

917
    $('#textgen_type').on('change', function () {
918
        const type = String($(this).val());
919
        textgenerationwebui_settings.type = type;
920

921
        if ([VLLM, APHRODITE, INFERMATICAI].includes(textgenerationwebui_settings.type)) {
922
            $('#mirostat_mode_textgenerationwebui').attr('step', 2); //Aphro disallows mode 1
923
            $('#do_sample_textgenerationwebui').prop('checked', true); //Aphro should always do sample; 'otherwise set temp to 0 to mimic no sample'
924
            $('#ban_eos_token_textgenerationwebui').prop('checked', false); //Aphro should not ban EOS, just ignore it; 'add token '2' to ban list do to this'
925
            //special handling for vLLM/Aphrodite topK -1 disable state
926
            $('#top_k_textgenerationwebui').attr('min', -1);
927
            if ($('#top_k_textgenerationwebui').val() === '0' || textgenerationwebui_settings.top_k === 0) {
928
                textgenerationwebui_settings.top_k = -1;
929
                $('#top_k_textgenerationwebui').val('-1').trigger('input');
930
            }
931
        } else {
932
            $('#mirostat_mode_textgenerationwebui').attr('step', 1);
933
            //undo special vLLM/Aphrodite setup for topK
934
            $('#top_k_textgenerationwebui').attr('min', 0);
935
            if ($('#top_k_textgenerationwebui').val() === '-1' || textgenerationwebui_settings.top_k === -1) {
936
                textgenerationwebui_settings.top_k = 0;
937
                $('#top_k_textgenerationwebui').val('0').trigger('input');
938
            }
939
        }
940

941
        showSamplerControls(type);
942
        setOnlineStatus('no_connection');
943
        BIAS_CACHE.delete(BIAS_KEY);
944

945
        $('#main_api').trigger('change');
946

947
        if (!SERVER_INPUTS[type] || textgenerationwebui_settings.server_urls[type]) {
948
            $('#api_button_textgenerationwebui').trigger('click');
949
        }
950

951
        saveSettingsDebounced();
952
    });
953

954
    $('#settings_preset_textgenerationwebui').on('change', async function () {
955
        const presetName = $(this).val();
956
        await selectPreset(presetName);
957
        await eventSource.emit(event_types.PRESET_CHANGED, { apiId: 'textgenerationwebui', name: presetName });
958
    });
959

960
    $('#samplerResetButton').off('click').on('click', function () {
961
        const inputs = {
962
            'temp_textgenerationwebui': 1,
963
            'top_k_textgenerationwebui': [INFERMATICAI, APHRODITE, VLLM].includes(textgenerationwebui_settings.type) ? -1 : 0,
964
            'top_p_textgenerationwebui': 1,
965
            'min_p_textgenerationwebui': 0,
966
            'rep_pen_textgenerationwebui': 1,
967
            'rep_pen_range_textgenerationwebui': 0,
968
            'rep_pen_decay_textgenerationwebui': 0,
969
            'dynatemp_textgenerationwebui': false,
970
            'seed_textgenerationwebui': -1,
971
            'ban_eos_token_textgenerationwebui': false,
972
            'do_sample_textgenerationwebui': true,
973
            'add_bos_token_textgenerationwebui': true,
974
            'temperature_last_textgenerationwebui': true,
975
            'skip_special_tokens_textgenerationwebui': true,
976
            'include_reasoning_textgenerationwebui': true,
977
            'top_a_textgenerationwebui': 0,
978
            'top_a_counter_textgenerationwebui': 0,
979
            'mirostat_mode_textgenerationwebui': 0,
980
            'mirostat_tau_textgenerationwebui': 5,
981
            'mirostat_eta_textgenerationwebui': 0.1,
982
            'tfs_textgenerationwebui': 1,
983
            'epsilon_cutoff_textgenerationwebui': 0,
984
            'eta_cutoff_textgenerationwebui': 0,
985
            'encoder_rep_pen_textgenerationwebui': 1,
986
            'freq_pen_textgenerationwebui': 0,
987
            'presence_pen_textgenerationwebui': 0,
988
            'skew_textgenerationwebui': 0,
989
            'no_repeat_ngram_size_textgenerationwebui': 0,
990
            'speculative_ngram_textgenerationwebui': false,
991
            'min_length_textgenerationwebui': 0,
992
            'num_beams_textgenerationwebui': 1,
993
            'length_penalty_textgenerationwebui': 1,
994
            'penalty_alpha_textgenerationwebui': 0,
995
            'typical_p_textgenerationwebui': 1, // Added entry
996
            'guidance_scale_textgenerationwebui': 1,
997
            'smoothing_factor_textgenerationwebui': 0,
998
            'smoothing_curve_textgenerationwebui': 1,
999
            'dry_allowed_length_textgenerationwebui': 2,
1000
            'dry_multiplier_textgenerationwebui': 0,
1001
            'dry_base_textgenerationwebui': 1.75,
1002
            'dry_penalty_last_n_textgenerationwebui': 0,
1003
            'xtc_threshold_textgenerationwebui': 0.1,
1004
            'xtc_probability_textgenerationwebui': 0,
1005
            'nsigma_textgenerationwebui': 0,
1006
            'min_keep_textgenerationwebui': 0,
1007
            'adaptive_target_textgenerationwebui': -0.01,
1008
            'adaptive_decay_textgenerationwebui': 0.9,
1009
        };
1010

1011
        for (const [id, value] of Object.entries(inputs)) {
1012
            const inputElement = $(`#${id}`);
1013
            const valueToSet = typeof value === 'boolean' ? String(value) : value;
1014
            if (inputElement.prop('type') === 'checkbox') {
1015
                inputElement.prop('checked', value).trigger('input');
1016
            } else if (inputElement.prop('type') === 'number') {
1017
                inputElement.val(valueToSet).trigger('input');
1018
            } else {
1019
                inputElement.val(valueToSet).trigger('input');
1020
                if (power_user.enableZenSliders) {
1021
                    let masterElementID = inputElement.prop('id');
1022
                    console.log(masterElementID);
1023
                    let zenSlider = $(`#${masterElementID}_zenslider`).slider();
1024
                    zenSlider.slider('option', 'value', value);
1025
                    zenSlider.slider('option', 'slide')
1026
                        .call(zenSlider, null, {
1027
                            handle: $('.ui-slider-handle', zenSlider), value: value,
1028
                        });
1029
                }
1030
            }
1031
        }
1032
    });
1033

1034
    for (const i of setting_names) {
1035
        $(`#${i}_textgenerationwebui`).attr('x-setting-id', i);
1036
        $(document).on('input', `#${i}_textgenerationwebui`, function () {
1037
            const isCheckbox = $(this).attr('type') == 'checkbox';
1038
            const isText = $(this).attr('type') == 'text' || $(this).is('textarea');
1039
            const id = $(this).attr('x-setting-id');
1040

1041
            if (isCheckbox) {
1042
                const value = $(this).prop('checked');
1043
                textgenerationwebui_settings[id] = value;
1044
            } else if (isText) {
1045
                const value = $(this).val();
1046
                textgenerationwebui_settings[id] = value;
1047
            } else {
1048
                const value = Number($(this).val());
1049
                $(`#${id}_counter_textgenerationwebui`).val(value);
1050
                textgenerationwebui_settings[id] = value;
1051
                //special handling for vLLM/Aphrodite using -1 as disabled instead of 0
1052
                if ($(this).attr('id') === 'top_k_textgenerationwebui' && [INFERMATICAI, APHRODITE, VLLM].includes(textgenerationwebui_settings.type) && value === 0) {
1053
                    textgenerationwebui_settings[id] = -1;
1054
                    $(this).val(-1);
1055
                }
1056
            }
1057
            saveSettingsDebounced();
1058
        });
1059
    }
1060

1061
    $('#textgen_logit_bias_new_entry').on('click', () => createNewLogitBiasEntry(textgenerationwebui_settings.logit_bias, BIAS_KEY));
1062

1063
    $('#openrouter_providers_text').on('change', function () {
1064
        const selectedProviders = $(this).val();
1065

1066
        // Not a multiple select?
1067
        if (!Array.isArray(selectedProviders)) {
1068
            return;
1069
        }
1070

1071
        textgenerationwebui_settings.openrouter_providers = selectedProviders;
1072

1073
        updateOpenRouterProvidersWarning('#openrouter_providers_text');
1074
        saveSettingsDebounced();
1075
    });
1076

1077
    $('#openrouter_allow_fallbacks_textgenerationwebui').on('input', function () {
1078
        updateOpenRouterProvidersWarning('#openrouter_providers_text');
1079
    });
1080

1081
    $('#openrouter_quantizations_text').on('change', function () {
1082
        const selectedQuantizations = $(this).val();
1083

1084
        // Not a multiple select?
1085
        if (!Array.isArray(selectedQuantizations)) {
1086
            return;
1087
        }
1088

1089
        textgenerationwebui_settings.openrouter_quantizations = selectedQuantizations;
1090

1091
        saveSettingsDebounced();
1092
    });
1093

1094
    $('#api_button_textgenerationwebui').on('click', async function (e) {
1095
        const keys = [
1096
            { id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },
1097
            { id: 'api_key_vllm', secret: SECRET_KEYS.VLLM },
1098
            { id: 'api_key_aphrodite', secret: SECRET_KEYS.APHRODITE },
1099
            { id: 'api_key_tabby', secret: SECRET_KEYS.TABBY },
1100
            { id: 'api_key_togetherai', secret: SECRET_KEYS.TOGETHERAI },
1101
            { id: 'api_key_ooba', secret: SECRET_KEYS.OOBA },
1102
            { id: 'api_key_infermaticai', secret: SECRET_KEYS.INFERMATICAI },
1103
            { id: 'api_key_dreamgen', secret: SECRET_KEYS.DREAMGEN },
1104
            { id: 'api_key_openrouter-tg', secret: SECRET_KEYS.OPENROUTER },
1105
            { id: 'api_key_koboldcpp', secret: SECRET_KEYS.KOBOLDCPP },
1106
            { id: 'api_key_llamacpp', secret: SECRET_KEYS.LLAMACPP },
1107
            { id: 'api_key_featherless', secret: SECRET_KEYS.FEATHERLESS },
1108
            { id: 'api_key_huggingface', secret: SECRET_KEYS.HUGGINGFACE },
1109
            { id: 'api_key_generic', secret: SECRET_KEYS.GENERIC },
1110
        ];
1111

1112
        for (const key of keys) {
1113
            const keyValue = String($(`#${key.id}`).val()).trim();
1114
            if (keyValue.length) {
1115
                await writeSecret(key.secret, keyValue);
1116
            }
1117
        }
1118

1119
        validateTextGenUrl();
1120
        startStatusLoading();
1121
        saveSettingsDebounced();
1122
        getStatusTextgen();
1123
    });
1124
}
1125

1126
/**
1127
 * Hides and shows preset samplers from the left panel.
1128
 * @param {string?} apiType API Type selected in API Connections - Currently selected one by default
1129
 * @returns void
1130
 */
1131
function showSamplerControls(apiType = null) {
1132
    $('#textgenerationwebui_api-settings [data-tg-samplers], #textgenerationwebui_api [data-tg-samplers]').each(function (idx, elem) {
1133
        const typeSpecificControlled = $(elem).data('tg-type') !== undefined;
1134

1135
        if (!typeSpecificControlled) $(this).show();
1136
    });
1137

1138
    showTypeSpecificControls(apiType ?? textgenerationwebui_settings.type);
1139

1140
    const prioritizeManualSamplerSelect = isSamplerManualPriorityEnabled(apiType ?? textgenerationwebui_settings.type);
1141
    const samplersActivatedManually = getActiveManualApiSamplers(apiType ?? textgenerationwebui_settings.type);
1142

1143
    if (!samplersActivatedManually?.length || !prioritizeManualSamplerSelect) return;
1144

1145
    $('#textgenerationwebui_api-settings [data-tg-samplers], #textgenerationwebui_api [data-tg-samplers]').each(function () {
1146
        const tgSamplers = $(this).attr('data-tg-samplers').split(',').map(x => x.trim()).filter(str => str !== '');
1147

1148
        for (const tgSampler of tgSamplers) {
1149
            if (samplersActivatedManually.includes(tgSampler)) {
1150
                $(this).show();
1151
                return;
1152
            } else {
1153
                $(this).hide();
1154
            }
1155
        }
1156
    });
1157
}
1158

1159
function showTypeSpecificControls(apiType) {
1160
    $('[data-tg-type]').each(function () {
1161
        const mode = String($(this).attr('data-tg-type-mode') ?? '').toLowerCase().trim();
1162
        const tgTypes = $(this).attr('data-tg-type').split(',').map(x => x.trim());
1163

1164
        if (mode === 'except') {
1165
            $(this)[tgTypes.includes(apiType) ? 'hide' : 'show']();
1166
            return;
1167
        }
1168

1169
        for (const tgType of tgTypes) {
1170
            if (tgType === apiType || tgType == 'all') {
1171
                $(this).show();
1172
                return;
1173
            } else {
1174
                $(this).hide();
1175
            }
1176
        }
1177
    });
1178
}
1179

1180
/**
1181
 * Inserts missing items from the source array into the target array.
1182
 * @param {any[]} source - Source array
1183
 * @param {any[]} target - Target array
1184
 * @returns {void}
1185
 */
1186
function insertMissingArrayItems(source, target) {
1187
    if (source === target || !Array.isArray(source) || !Array.isArray(target)) {
1188
        return;
1189
    }
1190

1191
    for (const item of source) {
1192
        if (!target.includes(item)) {
1193
            const index = source.indexOf(item);
1194
            target.splice(index, 0, item);
1195
        }
1196
    }
1197
}
1198

1199
function setSettingByName(setting, value, trigger) {
1200
    if ('extensions' === setting) {
1201
        value = value || {};
1202
        textgenerationwebui_settings.extensions = value;
1203
        return;
1204
    }
1205

1206
    if ('json_schema' === setting) {
1207
        textgenerationwebui_settings.json_schema = value ?? null;
1208
        $('#tabby_json_schema').val(value ? JSON.stringify(textgenerationwebui_settings.json_schema, null, 2) : '');
1209
        return;
1210
    }
1211

1212
    if (value === null || value === undefined) {
1213
        return;
1214
    }
1215

1216
    if ('sampler_order' === setting) {
1217
        value = Array.isArray(value) ? value : KOBOLDCPP_ORDER;
1218
        sortKoboldItemsByOrder(value);
1219
        textgenerationwebui_settings.sampler_order = value;
1220
        return;
1221
    }
1222

1223
    if ('sampler_priority' === setting) {
1224
        value = Array.isArray(value) ? value : OOBA_DEFAULT_ORDER;
1225
        insertMissingArrayItems(OOBA_DEFAULT_ORDER, value);
1226
        sortOobaItemsByOrder(value);
1227
        textgenerationwebui_settings.sampler_priority = value;
1228
        return;
1229
    }
1230

1231
    if ('samplers_priorities' === setting) {
1232
        value = Array.isArray(value) ? value : APHRODITE_DEFAULT_ORDER;
1233
        insertMissingArrayItems(APHRODITE_DEFAULT_ORDER, value);
1234
        sortAphroditeItemsByOrder(value);
1235
        textgenerationwebui_settings.samplers_priorities = value;
1236
        return;
1237
    }
1238

1239
    if ('samplers' === setting) {
1240
        value = Array.isArray(value) ? value : LLAMACPP_DEFAULT_ORDER;
1241
        insertMissingArrayItems(LLAMACPP_DEFAULT_ORDER, value);
1242
        sortLlamacppItemsByOrder(value);
1243
        textgenerationwebui_settings.samplers = value;
1244
        return;
1245
    }
1246

1247
    if ('logit_bias' === setting) {
1248
        textgenerationwebui_settings.logit_bias = Array.isArray(value) ? value : [];
1249
        return;
1250
    }
1251

1252
    const isCheckbox = $(`#${setting}_textgenerationwebui`).attr('type') == 'checkbox';
1253
    const isText = $(`#${setting}_textgenerationwebui`).attr('type') == 'text' || $(`#${setting}_textgenerationwebui`).is('textarea');
1254
    if (isCheckbox) {
1255
        const val = Boolean(value);
1256
        $(`#${setting}_textgenerationwebui`).prop('checked', val);
1257

1258
        if ('send_banned_tokens' === setting) {
1259
            $(`#${setting}_textgenerationwebui`).trigger('change');
1260
        }
1261
    } else if (isText) {
1262
        $(`#${setting}_textgenerationwebui`).val(value);
1263
    } else {
1264
        const val = parseFloat(value);
1265
        $(`#${setting}_textgenerationwebui`).val(val);
1266
        $(`#${setting}_counter_textgenerationwebui`).val(val);
1267
        if (power_user.enableZenSliders) {
1268
            let zenSlider = $(`#${setting}_textgenerationwebui_zenslider`).slider();
1269
            zenSlider.slider('option', 'value', val);
1270
            zenSlider.slider('option', 'slide')
1271
                .call(zenSlider, null, {
1272
                    handle: $('.ui-slider-handle', zenSlider), value: val,
1273
                });
1274
        }
1275
    }
1276

1277
    if (trigger) {
1278
        $(`#${setting}_textgenerationwebui`).trigger('input');
1279
    }
1280
}
1281

1282
/**
1283
 * Sends a streaming request for textgenerationwebui.
1284
 * @param {object} generate_data
1285
 * @param {AbortSignal} signal
1286
 * @returns {Promise<(function(): AsyncGenerator<{swipes: [], text: string, toolCalls: [], logprobs: {token: string, topLogprobs: Candidate[]}|null}, void, *>)|*>}
1287
 * @throws {Error} - If the response status is not OK, or from within the generator
1288
 */
1289
export async function generateTextGenWithStreaming(generate_data, signal) {
1290
    generate_data.stream = true;
1291

1292
    const response = await fetch('/api/backends/text-completions/generate', {
1293
        headers: {
1294
            ...getRequestHeaders(),
1295
        },
1296
        body: JSON.stringify(generate_data),
1297
        method: 'POST',
1298
        signal: signal,
1299
    });
1300

1301
    if (!response.ok) {
1302
        tryParseStreamingError(response, await response.text());
1303
        throw new Error(`Got response status ${response.status}`);
1304
    }
1305

1306
    const eventStream = getEventSourceStream();
1307
    response.body.pipeThrough(eventStream);
1308
    const reader = eventStream.readable.getReader();
1309

1310
    return async function* streamData() {
1311
        let text = '';
1312
        /** @type {import('./logprobs.js').TokenLogprobs | null} */
1313
        let logprobs = null;
1314
        const swipes = [];
1315
        const toolCalls = [];
1316
        const state = { reasoning: '' };
1317
        while (true) {
1318
            const { done, value } = await reader.read();
1319
            if (done) return;
1320
            if (value.data === '[DONE]') return;
1321

1322
            tryParseStreamingError(response, value.data);
1323

1324
            let data = JSON.parse(value.data);
1325

1326
            if (data?.choices?.[0]?.index > 0) {
1327
                const swipeIndex = data.choices[0].index - 1;
1328
                swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.choices[0].text;
1329
            } else if (data?.index > 0) {
1330
                // llama.cpp streaming swipe
1331
                const swipeIndex = data.index - 1;
1332
                swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.content;
1333
            } else {
1334
                const newText = data?.choices?.[0]?.text || data?.content || '';
1335
                text += newText;
1336
                logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs || data?.completion_probabilities);
1337
                state.reasoning += data?.choices?.[0]?.reasoning ?? data?.choices?.[0]?.thinking ?? '';
1338
            }
1339

1340
            yield { text, swipes, logprobs, toolCalls, state };
1341
        }
1342
    };
1343
}
1344

1345
/**
1346
 * parseTextgenLogprobs converts a logprobs object returned from a textgen API
1347
 * for a single token into a TokenLogprobs object used by the Token
1348
 * Probabilities feature.
1349
 * @param {string} token - the text of the token that the logprobs are for
1350
 * @param {Object} logprobs - logprobs object returned from the API
1351
 * @returns {import('./logprobs.js').TokenLogprobs | null} - converted logprobs
1352
 */
1353
export function parseTextgenLogprobs(token, logprobs) {
1354
    if (!logprobs) {
1355
        return null;
1356
    }
1357

1358
    switch (textgenerationwebui_settings.type) {
1359
        case KOBOLDCPP:
1360
        case TABBY:
1361
        case VLLM:
1362
        case APHRODITE:
1363
        case MANCER:
1364
        case INFERMATICAI:
1365
        case OOBA: {
1366
            /** @type {Record<string, number>[]} */
1367
            const topLogprobs = logprobs.top_logprobs;
1368
            if (!topLogprobs?.length) {
1369
                return null;
1370
            }
1371
            const candidates = Object.entries(topLogprobs[0]);
1372
            return { token, topLogprobs: candidates };
1373
        }
1374
        case LLAMACPP: {
1375
            if (!logprobs?.length) {
1376
                return null;
1377
            }
1378

1379
            // 3 cases:
1380
            // 1. Before commit 6c5bc06, "probs" key with "tok_str"/"prob", and probs are [0, 1] so use them directly.
1381
            // 2. After commit 6c5bc06 but before commit 89d604f broke logprobs (they all return the first token's logprobs)
1382
            //    We don't know the llama.cpp version so we can't do much about this.
1383
            // 3. After commit 89d604f uses OpenAI-compatible format with "completion_probabilities" and "token"/"logprob" keys.
1384
            //    Note that it is also the *actual* logprob (negative number), so we need to convert to [0, 1].
1385
            if (logprobs?.[0]?.probs) {
1386
                const candidates = logprobs?.[0]?.probs?.map(x => [x.tok_str, x.prob]);
1387
                if (!candidates) {
1388
                    return null;
1389
                }
1390
                return { token, topLogprobs: candidates };
1391
            } else if (logprobs?.[0].top_logprobs) {
1392
                const candidates = logprobs?.[0]?.top_logprobs?.map(x => [x.token, Math.exp(x.logprob)]);
1393
                if (!candidates) {
1394
                    return null;
1395
                }
1396
                return { token, topLogprobs: candidates };
1397
            }
1398
            return null;
1399
        }
1400
        default:
1401
            return null;
1402
    }
1403
}
1404

1405
export function parseTabbyLogprobs(data) {
1406
    const text = data?.choices?.[0]?.text;
1407
    const offsets = data?.choices?.[0]?.logprobs?.text_offset;
1408

1409
    if (!text || !offsets) {
1410
        return null;
1411
    }
1412

1413
    // Convert string offsets list to tokens
1414
    const tokens = offsets?.map((offset, index) => {
1415
        const nextOffset = offsets[index + 1] || text.length;
1416
        return text.substring(offset, nextOffset);
1417
    });
1418

1419
    const topLogprobs = data?.choices?.[0]?.logprobs?.top_logprobs?.map(x => ({ top_logprobs: [x] }));
1420
    return tokens?.map((token, index) => parseTextgenLogprobs(token, topLogprobs[index])) || null;
1421
}
1422

1423
/**
1424
 * Parses errors in streaming responses and displays them in toastr.
1425
 * @param {Response} response - Response from the server.
1426
 * @param {string} decoded - Decoded response body.
1427
 * @returns {void} Nothing.
1428
 * @throws {Error} If the response contains an error message, throws Error with the message.
1429
 */
1430
function tryParseStreamingError(response, decoded) {
1431
    let data = {};
1432

1433
    try {
1434
        data = JSON.parse(decoded);
1435
    } catch {
1436
        // No JSON. Do nothing.
1437
    }
1438

1439
    const message = data?.error?.message || data?.error || data?.message || data?.detail;
1440

1441
    if (message) {
1442
        toastr.error(message, 'Text Completion API');
1443
        throw new Error(message);
1444
    }
1445
}
1446

1447
/**
1448
 * Converts a string of comma-separated integers to an array of integers.
1449
 * @param {string} string Input string
1450
 * @returns {number[]} Array of integers
1451
 */
1452
function toIntArray(string) {
1453
    if (!string) {
1454
        return [];
1455
    }
1456

1457
    return string.split(',').map(x => parseInt(x)).filter(x => !isNaN(x));
1458
}
1459

1460
/**
1461
 * Gets the text generation model specified by the given text completion settings
1462
 * @param {TextCompletionSettings} settings Text completion settings to use
1463
 * @returns {string} model name
1464
 */
1465
export function getTextGenModel(settings = null) {
1466
    settings = settings ?? textgenerationwebui_settings;
1467
    switch (settings.type) {
1468
        case OOBA:
1469
            if (settings.custom_model) {
1470
                return settings.custom_model;
1471
            }
1472
            break;
1473
        case GENERIC:
1474
            if (settings.generic_model) {
1475
                return settings.generic_model;
1476
            }
1477
            break;
1478
        case MANCER:
1479
            return settings.mancer_model;
1480
        case TOGETHERAI:
1481
            return settings.togetherai_model;
1482
        case INFERMATICAI:
1483
            return settings.infermaticai_model;
1484
        case DREAMGEN:
1485
            return settings.dreamgen_model;
1486
        case OPENROUTER:
1487
            return settings.openrouter_model;
1488
        case VLLM:
1489
            return settings.vllm_model;
1490
        case APHRODITE:
1491
            return settings.aphrodite_model;
1492
        case OLLAMA:
1493
            if (!settings.ollama_model) {
1494
                toastr.error(t`No Ollama model selected.`, 'Text Completion API');
1495
                throw new Error('No Ollama model selected');
1496
            }
1497
            return settings.ollama_model;
1498
        case FEATHERLESS:
1499
            return settings.featherless_model;
1500
        case HUGGINGFACE:
1501
            return 'tgi';
1502
        case TABBY:
1503
            if (settings.tabby_model) {
1504
                return settings.tabby_model;
1505
            }
1506
            break;
1507
        case LLAMACPP:
1508
            if (settings.llamacpp_model) {
1509
                return settings.llamacpp_model;
1510
            }
1511
            break;
1512
        default:
1513
            return undefined;
1514
    }
1515

1516
    return undefined;
1517
}
1518

1519
export function isJsonSchemaSupported() {
1520
    return [TABBY, LLAMACPP].includes(textgenerationwebui_settings.type) && main_api === 'textgenerationwebui';
1521
}
1522

1523
/**
1524
 * Returns whether dynamic temperature is supported by the given text completion settings
1525
 * @param {TextCompletionSettings} settings Text completion settings to use
1526
 * @returns {boolean} Whether dynamic temperature supported
1527
 */
1528
function isDynamicTemperatureSupported(settings = null) {
1529
    settings = settings ?? textgenerationwebui_settings;
1530
    return settings.dynatemp && DYNATEMP_BLOCK?.dataset?.tgType?.includes(settings.type);
1531
}
1532

1533
/**
1534
 * Gets the number of logprobs to request based on the selected type.
1535
 * @param {string} type If it's set, ignores active type
1536
 * @returns {number} Number of logprobs to request
1537
 */
1538
export function getLogprobsNumber(type = null) {
1539
    const selectedType = type ?? textgenerationwebui_settings.type;
1540
    if (selectedType === VLLM || selectedType === INFERMATICAI) {
1541
        return 5;
1542
    }
1543

1544
    return 10;
1545
}
1546

1547
/**
1548
 * Replaces {{macro}} in a comma-separated or serialized JSON array string.
1549
 * @param {string} str Input string
1550
 * @returns {string} Output string
1551
 */
1552
export function replaceMacrosInList(str) {
1553
    if (!str || typeof str !== 'string') {
1554
        return str;
1555
    }
1556

1557
    try {
1558
        const array = JSON.parse(str);
1559
        if (!Array.isArray(array)) {
1560
            throw new Error('Not an array');
1561
        }
1562
        for (let i = 0; i < array.length; i++) {
1563
            array[i] = substituteParams(array[i]);
1564
        }
1565
        return JSON.stringify(array);
1566
    } catch {
1567
        const array = str.split(',');
1568
        for (let i = 0; i < array.length; i++) {
1569
            array[i] = substituteParams(array[i]);
1570
        }
1571
        return array.join(',');
1572
    }
1573
}
1574

1575
/**
1576
 * Build the generation parameter object for an text completion request
1577
 * @param {TextCompletionSettings} settings Text completion settings to use
1578
 * @param {string} model Model to use
1579
 * @param {string} finalPrompt The final prompt to send
1580
 * @param {number} maxTokens Max allowed generation tokens
1581
 * @param {boolean} isImpersonate Whether this is for an impersonation
1582
 * @param {boolean} isContinue Whether this is for a continue
1583
 * @param {object} cfgValues Additional parameters (guidanceScale, negativePrompt)
1584
 * @param {string} type Request type (impersonate, quiet, continue, etc)
1585
 * @returns {object} Final generation parameters object appropriate for the text completion source
1586
 */
1587
export function createTextGenGenerationData(settings, model, finalPrompt = null, maxTokens = null, isImpersonate = false, isContinue = false, cfgValues = null, type = 'quiet') {
1588
    settings = settings ?? textgenerationwebui_settings;
1589
    model = model ?? getTextGenModel(settings);
1590

1591
    const canMultiSwipe = !isContinue && !isImpersonate && type !== 'quiet';
1592
    const dynatemp = isDynamicTemperatureSupported(settings);
1593
    const { banned_tokens, banned_strings } = getCustomTokenBans(settings);
1594
    const jsonSchema = isObject(settings.json_schema)
1595
        ? settings.json_schema_allow_empty
1596
            ? settings.json_schema
1597
            : Object.keys(settings.json_schema).length > 0 ? settings.json_schema : undefined
1598
        : undefined;
1599

1600
    let params = {
1601
        'prompt': finalPrompt,
1602
        'model': model,
1603
        'max_new_tokens': maxTokens,
1604
        'max_tokens': maxTokens,
1605
        'logprobs': power_user.request_token_probabilities ? getLogprobsNumber(settings.type) : undefined,
1606
        'temperature': dynatemp ? (settings.min_temp + settings.max_temp) / 2 : settings.temp,
1607
        'top_p': settings.top_p,
1608
        'typical_p': settings.typical_p,
1609
        'typical': settings.typical_p,
1610
        'sampler_seed': settings.seed >= 0 ? settings.seed : undefined,
1611
        'min_p': settings.min_p,
1612
        'repetition_penalty': settings.rep_pen,
1613
        'frequency_penalty': settings.freq_pen,
1614
        'presence_penalty': settings.presence_pen,
1615
        'top_k': settings.top_k,
1616
        'skew': settings.skew,
1617
        'min_length': settings.type === OOBA ? settings.min_length : undefined,
1618
        'minimum_message_content_tokens': settings.type === DREAMGEN ? settings.min_length : undefined,
1619
        'min_tokens': settings.min_length,
1620
        'num_beams': settings.type === OOBA ? settings.num_beams : undefined,
1621
        'length_penalty': settings.type === OOBA ? settings.length_penalty : undefined,
1622
        'early_stopping': settings.type === OOBA ? settings.early_stopping : undefined,
1623
        'add_bos_token': settings.add_bos_token,
1624
        'dynamic_temperature': dynatemp ? true : undefined,
1625
        'dynatemp_low': dynatemp ? settings.min_temp : undefined,
1626
        'dynatemp_high': dynatemp ? settings.max_temp : undefined,
1627
        'dynatemp_range': dynatemp ? (settings.max_temp - settings.min_temp) / 2 : undefined,
1628
        'dynatemp_exponent': dynatemp ? settings.dynatemp_exponent : undefined,
1629
        'smoothing_factor': settings.smoothing_factor,
1630
        'smoothing_curve': settings.smoothing_curve,
1631
        'dry_allowed_length': settings.dry_allowed_length,
1632
        'dry_multiplier': settings.dry_multiplier,
1633
        'dry_base': settings.dry_base,
1634
        'dry_sequence_breakers': replaceMacrosInList(settings.dry_sequence_breakers),
1635
        'dry_penalty_last_n': settings.dry_penalty_last_n,
1636
        'max_tokens_second': settings.max_tokens_second,
1637
        'sampler_priority': settings.type === OOBA ? settings.sampler_priority : undefined,
1638
        'samplers': settings.type === LLAMACPP ? settings.samplers : undefined,
1639
        'stopping_strings': getStoppingStrings(isImpersonate, isContinue),
1640
        'stop': getStoppingStrings(isImpersonate, isContinue),
1641
        'truncation_length': max_context,
1642
        'ban_eos_token': settings.ban_eos_token,
1643
        'skip_special_tokens': settings.skip_special_tokens,
1644
        'include_reasoning': settings.include_reasoning,
1645
        'top_a': settings.top_a,
1646
        'tfs': settings.tfs,
1647
        'epsilon_cutoff': [OOBA, MANCER].includes(settings.type) ? settings.epsilon_cutoff : undefined,
1648
        'eta_cutoff': [OOBA, MANCER].includes(settings.type) ? settings.eta_cutoff : undefined,
1649
        'mirostat_mode': settings.mirostat_mode,
1650
        'mirostat_tau': settings.mirostat_tau,
1651
        'mirostat_eta': settings.mirostat_eta,
1652
        'custom_token_bans': [APHRODITE, MANCER].includes(settings.type) ?
1653
            toIntArray(banned_tokens) :
1654
            banned_tokens,
1655
        'banned_strings': banned_strings,
1656
        'api_type': settings.type,
1657
        'api_server': getTextGenServer(settings.type),
1658
        'sampler_order': settings.type === textgen_types.KOBOLDCPP ? settings.sampler_order : undefined,
1659
        'xtc_threshold': settings.xtc_threshold,
1660
        'xtc_probability': settings.xtc_probability,
1661
        'nsigma': settings.nsigma,
1662
        'top_n_sigma': settings.nsigma,
1663
        'min_keep': settings.min_keep,
1664
        'adaptive_target': settings.adaptive_target,
1665
        'adaptive_decay': settings.adaptive_decay,
1666
        parseSequenceBreakers: function () {
1667
            try {
1668
                return JSON.parse(this.dry_sequence_breakers);
1669
            } catch {
1670
                if (typeof this.dry_sequence_breakers === 'string') {
1671
                    return this.dry_sequence_breakers.split(',');
1672
                }
1673
                return undefined;
1674
            }
1675
        },
1676
    };
1677
    const nonAphroditeParams = {
1678
        'rep_pen': settings.rep_pen,
1679
        'rep_pen_range': settings.rep_pen_range,
1680
        'repetition_decay': settings.type === TABBY ? settings.rep_pen_decay : undefined,
1681
        'repetition_penalty_range': settings.rep_pen_range,
1682
        'encoder_repetition_penalty': settings.type === OOBA ? settings.encoder_rep_pen : undefined,
1683
        'no_repeat_ngram_size': settings.type === OOBA ? settings.no_repeat_ngram_size : undefined,
1684
        'penalty_alpha': settings.type === OOBA ? settings.penalty_alpha : undefined,
1685
        'temperature_last': (settings.type === OOBA || settings.type === APHRODITE || settings.type == TABBY) ? settings.temperature_last : undefined,
1686
        'speculative_ngram': settings.type === TABBY ? settings.speculative_ngram : undefined,
1687
        'do_sample': settings.type === OOBA ? settings.do_sample : undefined,
1688
        'seed': settings.seed >= 0 ? settings.seed : undefined,
1689
        'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1,
1690
        'negative_prompt': cfgValues?.negativePrompt ?? substituteParams(settings.negative_prompt) ?? '',
1691
        'grammar_string': settings.grammar_string || undefined,
1692
        'json_schema': [TABBY, LLAMACPP].includes(settings.type) ? jsonSchema : undefined,
1693
        // llama.cpp aliases. In case someone wants to use LM Studio as Text Completion API
1694
        'repeat_penalty': settings.rep_pen,
1695
        'repeat_last_n': settings.rep_pen_range,
1696
        'n_predict': maxTokens,
1697
        'num_predict': maxTokens,
1698
        'num_ctx': max_context,
1699
        'mirostat': settings.mirostat_mode,
1700
        'ignore_eos': settings.ban_eos_token,
1701
        'n_probs': power_user.request_token_probabilities ? 10 : undefined,
1702
        'rep_pen_slope': settings.rep_pen_slope,
1703
    };
1704
    const vllmParams = {
1705
        'n': canMultiSwipe ? settings.n : 1,
1706
        'ignore_eos': settings.ignore_eos_token,
1707
        'spaces_between_special_tokens': settings.spaces_between_special_tokens,
1708
        'seed': settings.seed >= 0 ? settings.seed : undefined,
1709
    };
1710
    const aphroditeParams = {
1711
        'n': canMultiSwipe ? settings.n : 1,
1712
        'frequency_penalty': settings.freq_pen,
1713
        'presence_penalty': settings.presence_pen,
1714
        'repetition_penalty': settings.rep_pen,
1715
        'seed': settings.seed >= 0 ? settings.seed : undefined,
1716
        'stop': getStoppingStrings(isImpersonate, isContinue),
1717
        'temperature': dynatemp ? (settings.min_temp + settings.max_temp) / 2 : settings.temp,
1718
        'temperature_last': settings.temperature_last,
1719
        'top_p': settings.top_p,
1720
        'top_k': settings.top_k,
1721
        'top_a': settings.top_a,
1722
        'min_p': settings.min_p,
1723
        'tfs': settings.tfs,
1724
        'eta_cutoff': settings.eta_cutoff,
1725
        'epsilon_cutoff': settings.epsilon_cutoff,
1726
        'typical_p': settings.typical_p,
1727
        'smoothing_factor': settings.smoothing_factor,
1728
        'smoothing_curve': settings.smoothing_curve,
1729
        'ignore_eos': settings.ignore_eos_token,
1730
        'min_tokens': settings.min_length,
1731
        'skip_special_tokens': settings.skip_special_tokens,
1732
        'spaces_between_special_tokens': settings.spaces_between_special_tokens,
1733
        'guided_grammar': settings.grammar_string || undefined,
1734
        'guided_json': jsonSchema || undefined,
1735
        'early_stopping': false, // hacks
1736
        'include_stop_str_in_output': false,
1737
        'dynatemp_min': dynatemp ? settings.min_temp : undefined,
1738
        'dynatemp_max': dynatemp ? settings.max_temp : undefined,
1739
        'dynatemp_exponent': dynatemp ? settings.dynatemp_exponent : undefined,
1740
        'xtc_threshold': settings.xtc_threshold,
1741
        'xtc_probability': settings.xtc_probability,
1742
        'nsigma': settings.nsigma,
1743
        'custom_token_bans': toIntArray(banned_tokens),
1744
        'no_repeat_ngram_size': settings.no_repeat_ngram_size,
1745
        'sampler_priority': settings.type === APHRODITE && !arraysEqual(
1746
            settings.samplers_priorities,
1747
            APHRODITE_DEFAULT_ORDER)
1748
            ? settings.samplers_priorities
1749
            : undefined,
1750
    };
1751

1752
    if (settings.type === OPENROUTER) {
1753
        params.provider = settings.openrouter_providers;
1754
        params.quantizations = settings.openrouter_quantizations;
1755
        params.allow_fallbacks = settings.openrouter_allow_fallbacks;
1756
    }
1757

1758
    if (settings.type === KOBOLDCPP) {
1759
        params.grammar = settings.grammar_string || undefined;
1760
        params.grammar_retain_state = (settings.grammar_string && !!isContinue) ? true : undefined;
1761
        params.trim_stop = true;
1762
        params.dry_sequence_breakers = params.parseSequenceBreakers();
1763
    }
1764

1765
    if (settings.type === HUGGINGFACE) {
1766
        params.top_p = Math.min(Math.max(Number(params.top_p), 0.0), 0.999);
1767
        params.stop = Array.isArray(params.stop) ? params.stop.slice(0, 4) : [];
1768
        nonAphroditeParams.seed = settings.seed >= 0 ? settings.seed : Math.floor(Math.random() * Math.pow(2, 32));
1769
    }
1770

1771
    if (settings.type === MANCER) {
1772
        params.n = canMultiSwipe ? settings.n : 1;
1773
        params.epsilon_cutoff /= 1000;
1774
        params.eta_cutoff /= 1000;
1775
        params.dynatemp_mode = params.dynamic_temperature ? 1 : 0;
1776
        params.dynatemp_min = params.dynatemp_low;
1777
        params.dynatemp_max = params.dynatemp_high;
1778
        delete params.dynatemp_low;
1779
        delete params.dynatemp_high;
1780
        params.dry_sequence_breakers = params.parseSequenceBreakers();
1781
    }
1782

1783
    if (settings.type === TABBY || settings.type === LLAMACPP) {
1784
        params.n = canMultiSwipe ? settings.n : 1;
1785
    }
1786

1787
    switch (settings.type) {
1788
        case VLLM:
1789
        case INFERMATICAI:
1790
            params = Object.assign(params, vllmParams);
1791
            break;
1792

1793
        case APHRODITE:
1794
            // set params to aphroditeParams
1795
            params = Object.assign(params, aphroditeParams);
1796
            break;
1797

1798
        default:
1799
            params = Object.assign(params, nonAphroditeParams);
1800
            break;
1801
    }
1802

1803
    if (Array.isArray(settings.logit_bias) && settings.logit_bias.length) {
1804
        const logitBias = BIAS_CACHE.get(BIAS_KEY) || calculateLogitBias(settings);
1805
        BIAS_CACHE.set(BIAS_KEY, logitBias);
1806
        params.logit_bias = logitBias;
1807
    }
1808

1809
    if (settings.type === LLAMACPP || settings.type === OLLAMA) {
1810
        // Convert bias and token bans to array of arrays
1811
        const logitBiasArray = (params.logit_bias && typeof params.logit_bias === 'object' && Object.keys(params.logit_bias).length > 0)
1812
            ? Object.entries(params.logit_bias).map(([key, value]) => [Number(key), value])
1813
            : [];
1814
        const tokenBans = toIntArray(banned_tokens);
1815
        logitBiasArray.push(...tokenBans.map(x => [Number(x), false]));
1816
        const sequenceBreakers = params.parseSequenceBreakers();
1817
        const llamaCppParams = {
1818
            'logit_bias': logitBiasArray,
1819
            // Conflicts with ooba's grammar_string
1820
            'grammar': settings.grammar_string,
1821
            'cache_prompt': true,
1822
            'dry_sequence_breakers': sequenceBreakers,
1823
        };
1824
        params = Object.assign(params, llamaCppParams);
1825
        if (!Array.isArray(sequenceBreakers) || sequenceBreakers.length === 0) {
1826
            delete params.dry_sequence_breakers;
1827
        }
1828
    }
1829

1830
    // Grammar conflicts with with json_schema
1831
    if ([LLAMACPP, APHRODITE].includes(settings.type)) {
1832
        if (jsonSchema) {
1833
            delete params.grammar_string;
1834
            delete params.grammar;
1835
            delete params.guided_grammar;
1836
        } else {
1837
            delete params.json_schema;
1838
            delete params.guided_json;
1839
        }
1840
    }
1841
    return params;
1842
}
1843

1844
export async function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) {
1845
    const model = getTextGenModel(textgenerationwebui_settings);
1846
    const params = createTextGenGenerationData(textgenerationwebui_settings, model, finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type);
1847
    await eventSource.emit(event_types.TEXT_COMPLETION_SETTINGS_READY, params);
1848
    return params;
1849
}