Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · 51ad27fb · · 535 lines (17.8 KB)
2 contributors 
1import {


2    getRequestHeaders,


3    saveSettingsDebounced,


4    getStoppingStrings,


5    substituteParams,


6    setOnlineStatus,


7    resultCheckStatus,


8    main_api,


9    online_status,


10    abortStatusCheck,


11    startStatusLoading,


12    setGenerationParamsFromPreset,


13    eventSource,


14    event_types,


15} from '../script.js';


16import { t } from './i18n.js';


17import { autoSelectInstructPreset } from './instruct-mode.js';


18


19import {


20    power_user,


21} from './power-user.js';


22import { getEventSourceStream } from './sse-stream.js';


23import { getSortableDelay, versionCompare } from './utils.js';


24


25export let koboldai_settings;


26export let koboldai_setting_names;


27


28export const kai_settings = {


29    temp: 1,


30    rep_pen: 1,


31    rep_pen_range: 0,


32    top_p: 1,


33    min_p: 0,


34    top_a: 1,


35    top_k: 0,


36    typical: 1,


37    tfs: 1,


38    rep_pen_slope: 0.9,


39    streaming_kobold: false,


40    sampler_order: [0, 1, 2, 3, 4, 5, 6],


41    mirostat: 0,


42    mirostat_tau: 5.0,


43    mirostat_eta: 0.1,


44    use_default_badwordsids: false,


45    grammar: '',


46    seed: -1,


47    api_server: '',


48    preset_settings: 'gui',


49    extensions: {},


50};


51


52/**


53 * Stable version of KoboldAI has a nasty payload validation.


54 * It will reject any payload that has a key that is not in the whitelist.


55 * @typedef {Object.<string, boolean>} kai_flags


56 */


57export const kai_flags = {


58    can_use_tokenization: false,


59    can_use_stop_sequence: false,


60    can_use_streaming: false,


61    can_use_default_badwordsids: false,


62    can_use_mirostat: false,


63    can_use_grammar: false,


64    can_use_min_p: false,


65};


66


67const defaultValues = Object.freeze(structuredClone(kai_settings));


68


69const MIN_STOP_SEQUENCE_VERSION = '1.2.2';


70const MIN_UNBAN_VERSION = '1.2.4';


71const MIN_STREAMING_KCPPVERSION = '1.30';


72const MIN_TOKENIZATION_KCPPVERSION = '1.41';


73const MIN_MIROSTAT_KCPPVERSION = '1.35';


74const MIN_GRAMMAR_KCPPVERSION = '1.44';


75const MIN_MIN_P_KCPPVERSION = '1.48';


76const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];


77


78export function formatKoboldUrl(value) {


79    try {


80        const url = new URL(value);


81        if (!power_user.relaxed_api_urls) {


82            url.pathname = '/api';


83        }


84        return url.toString();


85    } catch {


86        // Just using URL as a validation check


87    }


88    return null;


89}


90


91function selectKoboldGuiPreset() {


92    $('#settings_preset option[value=gui]')


93        .attr('selected', 'true')


94        .trigger('change');


95}


96


97export function loadKoboldSettings(data, preset, settings) {


98    koboldai_setting_names = data.koboldai_setting_names;


99    koboldai_settings = data.koboldai_settings;


100    koboldai_settings.forEach(function (item, i, arr) {


101        koboldai_settings[i] = JSON.parse(item);


102    });


103


104    $('#settings_preset').empty();


105    $('#settings_preset').append('<option value="gui">GUI KoboldAI Settings</option>');


106    const names = {};


107    koboldai_setting_names.forEach(function (item, i, arr) {


108        names[item] = i;


109        $('#settings_preset').append(`<option value=${i}>${item}</option>`);


110    });


111    koboldai_setting_names = names;


112


113    kai_settings.preset_settings = preset.preset_settings ?? settings.preset_settings;


114    kai_settings.api_server = preset.api_server ?? settings.api_server;


115


116    if (kai_settings.preset_settings == 'gui') {


117        selectKoboldGuiPreset();


118    } else {


119        if (typeof koboldai_setting_names[kai_settings.preset_settings] !== 'undefined') {


120            $(`#settings_preset option[value=${koboldai_setting_names[kai_settings.preset_settings]}]`)


121                .attr('selected', 'true');


122        } else {


123            kai_settings.preset_settings = 'gui';


124            selectKoboldGuiPreset();


125        }


126    }


127


128    loadKoboldSettingsFromPreset(preset);


129


130    //Load the API server URL from settings


131    $('#api_url_text').val(kai_settings.api_server);


132}


133


134function loadKoboldSettingsFromPreset(preset) {


135    for (const name of Object.keys(kai_settings)) {


136        if (name === 'extensions') {


137            kai_settings.extensions = preset.extensions || {};


138            continue;


139        }


140


141        const value = preset[name] ?? defaultValues[name];


142        const slider = sliders.find(x => x.name === name);


143


144        if (!slider) {


145            continue;


146        }


147


148        const formattedValue = slider.format(value);


149        slider.setValue(value);


150        $(slider.sliderId).val(value);


151        $(slider.counterId).val(formattedValue);


152    }


153


154    if (Object.hasOwn(preset, 'streaming_kobold')) {


155        kai_settings.streaming_kobold = preset.streaming_kobold;


156        $('#streaming_kobold').prop('checked', kai_settings.streaming_kobold);


157    }


158    if (Object.hasOwn(preset, 'use_default_badwordsids')) {


159        kai_settings.use_default_badwordsids = preset.use_default_badwordsids;


160        $('#use_default_badwordsids').prop('checked', kai_settings.use_default_badwordsids);


161    }


162}


163


164/**


165 * Gets the Kobold generation data.


166 * @param {string} finalPrompt Final text prompt.


167 * @param {object} settings Settings preset object.


168 * @param {number} maxLength Maximum length.


169 * @param {number} maxContextLength Maximum context length.


170 * @param {boolean} isHorde True if the generation is for a horde, false otherwise.


171 * @param {string} type Generation type.


172 * @returns {object} Kobold generation data.


173 */


174export function getKoboldGenerationData(finalPrompt, settings, maxLength, maxContextLength, isHorde, type) {


175    const isImpersonate = type === 'impersonate';


176    const isContinue = type === 'continue';


177    const sampler_order = kai_settings.sampler_order || settings.sampler_order;


178


179    let generate_data = {


180        prompt: finalPrompt,


181        gui_settings: false,


182        sampler_order: sampler_order,


183        max_context_length: Number(maxContextLength),


184        max_length: maxLength,


185        rep_pen: Number(kai_settings.rep_pen),


186        rep_pen_range: Number(kai_settings.rep_pen_range),


187        rep_pen_slope: kai_settings.rep_pen_slope,


188        temperature: Number(kai_settings.temp),


189        tfs: kai_settings.tfs,


190        top_a: kai_settings.top_a,


191        top_k: kai_settings.top_k,


192        top_p: kai_settings.top_p,


193        min_p: (kai_flags.can_use_min_p || isHorde) ? kai_settings.min_p : undefined,


194        typical: kai_settings.typical,


195        use_world_info: false,


196        singleline: false,


197        stop_sequence: (kai_flags.can_use_stop_sequence || isHorde) ? getStoppingStrings(isImpersonate, isContinue) : undefined,


198        streaming: kai_settings.streaming_kobold && kai_flags.can_use_streaming && type !== 'quiet',


199        can_abort: kai_flags.can_use_streaming,


200        mirostat: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat : undefined,


201        mirostat_tau: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat_tau : undefined,


202        mirostat_eta: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat_eta : undefined,


203        use_default_badwordsids: (kai_flags.can_use_default_badwordsids || isHorde) ? kai_settings.use_default_badwordsids : undefined,


204        grammar: (kai_flags.can_use_grammar || isHorde) ? substituteParams(kai_settings.grammar) : undefined,


205        grammar_retain_state: (kai_flags.can_use_grammar && !!isContinue) ? true : undefined,


206        sampler_seed: kai_settings.seed >= 0 ? kai_settings.seed : undefined,


207        api_server: kai_settings.api_server,


208    };


209    return generate_data;


210}


211


212function tryParseStreamingError(response, decoded) {


213    try {


214        const data = JSON.parse(decoded);


215


216        if (!data) {


217            return;


218        }


219


220        if (data.error) {


221            toastr.error(data.error.message || response.statusText, 'KoboldAI API');


222            throw new Error(data);


223        }


224    } catch {


225        // No JSON. Do nothing.


226    }


227}


228


229export async function generateKoboldWithStreaming(generate_data, signal) {


230    const response = await fetch('/api/backends/kobold/generate', {


231        headers: getRequestHeaders(),


232        body: JSON.stringify(generate_data),


233        method: 'POST',


234        signal: signal,


235    });


236    if (!response.ok) {


237        tryParseStreamingError(response, await response.text());


238        throw new Error(`Got response status ${response.status}`);


239    }


240    const eventStream = getEventSourceStream();


241    response.body.pipeThrough(eventStream);


242    const reader = eventStream.readable.getReader();


243


244    return async function* streamData() {


245        let text = '';


246        while (true) {


247            const { done, value } = await reader.read();


248            if (done) return;


249


250            const data = JSON.parse(value.data);


251            if (data?.token) {


252                text += data.token;


253            }


254            yield { text, swipes: [], toolCalls: [], state: {} };


255        }


256    };


257}


258


259const sliders = [


260    {


261        name: 'temp',


262        sliderId: '#temp',


263        counterId: '#temp_counter',


264        format: (val) => Number(val).toFixed(2),


265        setValue: (val) => { kai_settings.temp = Number(val); },


266    },


267    {


268        name: 'rep_pen',


269        sliderId: '#rep_pen',


270        counterId: '#rep_pen_counter',


271        format: (val) => Number(val).toFixed(2),


272        setValue: (val) => { kai_settings.rep_pen = Number(val); },


273    },


274    {


275        name: 'rep_pen_range',


276        sliderId: '#rep_pen_range',


277        counterId: '#rep_pen_range_counter',


278        format: (val) => val,


279        setValue: (val) => { kai_settings.rep_pen_range = Number(val); },


280    },


281    {


282        name: 'top_p',


283        sliderId: '#top_p',


284        counterId: '#top_p_counter',


285        format: (val) => val,


286        setValue: (val) => { kai_settings.top_p = Number(val); },


287    },


288    {


289        name: 'min_p',


290        sliderId: '#min_p',


291        counterId: '#min_p_counter',


292        format: (val) => val,


293        setValue: (val) => { kai_settings.min_p = Number(val); },


294    },


295    {


296        name: 'top_a',


297        sliderId: '#top_a',


298        counterId: '#top_a_counter',


299        format: (val) => val,


300        setValue: (val) => { kai_settings.top_a = Number(val); },


301    },


302    {


303        name: 'top_k',


304        sliderId: '#top_k',


305        counterId: '#top_k_counter',


306        format: (val) => val,


307        setValue: (val) => { kai_settings.top_k = Number(val); },


308    },


309    {


310        name: 'typical',


311        sliderId: '#typical_p',


312        counterId: '#typical_p_counter',


313        format: (val) => val,


314        setValue: (val) => { kai_settings.typical = Number(val); },


315    },


316    {


317        name: 'tfs',


318        sliderId: '#tfs',


319        counterId: '#tfs_counter',


320        format: (val) => val,


321        setValue: (val) => { kai_settings.tfs = Number(val); },


322    },


323    {


324        name: 'rep_pen_slope',


325        sliderId: '#rep_pen_slope',


326        counterId: '#rep_pen_slope_counter',


327        format: (val) => val,


328        setValue: (val) => { kai_settings.rep_pen_slope = Number(val); },


329    },


330    {


331        name: 'sampler_order',


332        sliderId: '#no_op_selector',


333        counterId: '#no_op_selector',


334        format: (val) => val,


335        setValue: (val) => { sortItemsByOrder(val); kai_settings.sampler_order = val; },


336    },


337    {


338        name: 'mirostat',


339        sliderId: '#mirostat_mode_kobold',


340        counterId: '#mirostat_mode_counter_kobold',


341        format: (val) => val,


342        setValue: (val) => { kai_settings.mirostat = Number(val); },


343    },


344    {


345        name: 'mirostat_tau',


346        sliderId: '#mirostat_tau_kobold',


347        counterId: '#mirostat_tau_counter_kobold',


348        format: (val) => val,


349        setValue: (val) => { kai_settings.mirostat_tau = Number(val); },


350    },


351    {


352        name: 'mirostat_eta',


353        sliderId: '#mirostat_eta_kobold',


354        counterId: '#mirostat_eta_counter_kobold',


355        format: (val) => val,


356        setValue: (val) => { kai_settings.mirostat_eta = Number(val); },


357    },


358    {


359        name: 'grammar',


360        sliderId: '#grammar',


361        counterId: '#grammar_counter_kobold',


362        format: (val) => val,


363        setValue: (val) => { kai_settings.grammar = val; },


364    },


365    {


366        name: 'seed',


367        sliderId: '#seed_kobold',


368        counterId: '#seed_counter_kobold',


369        format: (val) => val,


370        setValue: (val) => { kai_settings.seed = Number(val); },


371    },


372];


373


374/**


375 * Sets the supported feature flags for the KoboldAI backend.


376 * @param {string} koboldUnitedVersion Kobold United version


377 * @param {string} koboldCppVersion KoboldCPP version


378 */


379export function setKoboldFlags(koboldUnitedVersion, koboldCppVersion) {


380    kai_flags.can_use_stop_sequence = versionCompare(koboldUnitedVersion, MIN_STOP_SEQUENCE_VERSION);


381    kai_flags.can_use_streaming = versionCompare(koboldCppVersion, MIN_STREAMING_KCPPVERSION);


382    kai_flags.can_use_tokenization = versionCompare(koboldCppVersion, MIN_TOKENIZATION_KCPPVERSION);


383    kai_flags.can_use_default_badwordsids = versionCompare(koboldUnitedVersion, MIN_UNBAN_VERSION);


384    kai_flags.can_use_mirostat = versionCompare(koboldCppVersion, MIN_MIROSTAT_KCPPVERSION);


385    kai_flags.can_use_grammar = versionCompare(koboldCppVersion, MIN_GRAMMAR_KCPPVERSION);


386    kai_flags.can_use_min_p = versionCompare(koboldCppVersion, MIN_MIN_P_KCPPVERSION);


387    const isKoboldCpp = versionCompare(koboldCppVersion, '1.0.0');


388    $('#koboldcpp_hint').toggleClass('displayNone', !isKoboldCpp);


389}


390


391/**


392 * Sorts the sampler items by the given order.


393 * @param {any[]} orderArray Sampler order array.


394 */


395function sortItemsByOrder(orderArray) {


396    console.debug('Preset samplers order: ' + orderArray);


397    const $draggableItems = $('#kobold_order');


398


399    for (let i = 0; i < orderArray.length; i++) {


400        const index = orderArray[i];


401        const $item = $draggableItems.find(`[data-id="${index}"]`).detach();


402        $draggableItems.append($item);


403    }


404}


405


406export async function getStatusKobold() {


407    let endpoint = kai_settings.api_server;


408


409    if (!endpoint) {


410        console.warn('No endpoint for status check');


411        setOnlineStatus('no_connection');


412        return resultCheckStatus();


413    }


414


415    try {


416        const response = await fetch('/api/backends/kobold/status', {


417            method: 'POST',


418            headers: getRequestHeaders(),


419            body: JSON.stringify({


420                main_api,


421                api_server: endpoint,


422            }),


423            signal: abortStatusCheck.signal,


424        });


425


426        const data = await response.json();


427


428        setOnlineStatus(data?.model ?? 'no_connection');


429


430        if (!data.koboldUnitedVersion) {


431            throw new Error(`Missing mandatory Kobold version in data: ${JSON.stringify(data)}`);


432        }


433


434        // Determine instruct mode preset


435        autoSelectInstructPreset(online_status);


436


437        // determine if we can use stop sequence and streaming


438        setKoboldFlags(data.koboldUnitedVersion, data.koboldCppVersion);


439


440        // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.


441        if (online_status === 'no_connection' && data.response) {


442            toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });


443        }


444    } catch (err) {


445        console.error('Error getting status', err);


446        setOnlineStatus('no_connection');


447    }


448


449    return resultCheckStatus();


450}


451


452export function initKoboldSettings() {


453    sliders.forEach(slider => {


454        $(document).on('input', slider.sliderId, function () {


455            const value = $(this).val();


456            const formattedValue = slider.format(value);


457            slider.setValue(value);


458            $(slider.counterId).val(formattedValue);


459            saveSettingsDebounced();


460        });


461    });


462


463    $('#api_button').on('click', function (e) {


464        if ($('#api_url_text').val() != '') {


465            const value = formatKoboldUrl(String($('#api_url_text').val()).trim());


466


467            if (!value) {


468                toastr.error('Please enter a valid URL.');


469                return;


470            }


471


472            $('#api_url_text').val(value);


473            kai_settings.api_server = value;


474            startStatusLoading();


475            saveSettingsDebounced();


476            getStatusKobold();


477        }


478    });


479


480    $('#streaming_kobold').on('input', function () {


481        const value = !!$(this).prop('checked');


482        kai_settings.streaming_kobold = value;


483        saveSettingsDebounced();


484    });


485


486    $('#use_default_badwordsids').on('input', function () {


487        const value = !!$(this).prop('checked');


488        kai_settings.use_default_badwordsids = value;


489        saveSettingsDebounced();


490    });


491


492    $('#kobold_order').sortable({


493        delay: getSortableDelay(),


494        stop: function () {


495            const order = [];


496            $('#kobold_order').children().each(function () {


497                order.push($(this).data('id'));


498            });


499            kai_settings.sampler_order = order;


500            console.log('Samplers reordered:', kai_settings.sampler_order);


501            saveSettingsDebounced();


502        },


503    });


504


505    $('#samplers_order_recommended').on('click', function () {


506        kai_settings.sampler_order = KOBOLDCPP_ORDER;


507        sortItemsByOrder(kai_settings.sampler_order);


508        saveSettingsDebounced();


509    });


510


511    $('#settings_preset').on('change', async function () {


512        if ($('#settings_preset').find(':selected').val() != 'gui') {


513            kai_settings.preset_settings = $('#settings_preset').find(':selected').text();


514            const preset = koboldai_settings[koboldai_setting_names[kai_settings.preset_settings]];


515            loadKoboldSettingsFromPreset(preset);


516            setGenerationParamsFromPreset(preset);


517            $('#kobold_api-settings').find('input').prop('disabled', false);


518            $('#kobold_api-settings').css('opacity', 1.0);


519            $('#kobold_order')


520                .css('opacity', 1)


521                .sortable('enable');


522        } else {


523            kai_settings.preset_settings = 'gui';


524


525            $('#kobold_api-settings').find('input').prop('disabled', true);


526            $('#kobold_api-settings').css('opacity', 0.5);


527


528            $('#kobold_order')


529                .css('opacity', 0.5)


530                .sortable('disable');


531        }


532        saveSettingsDebounced();


533        await eventSource.emit(event_types.PRESET_CHANGED, { apiId: 'kobold', name: kai_settings.preset_settings });


534    });


535}