Update llama.cpp textgen settings * Add min_keep, a llama.cpp-exclusive setting for constraining the effect of truncation samplers * Enable nsigma for llama.cpp, and add llama.cpp alias top_n_sigma, add nsigma to the llama.cpp sampler order block * Allow a negative value of nsigma as this represents 'disabled' in llama.cpp (while 0 is deterministic) * Remove tfs and top_a as these are not supported by llama.cpp (tfs was removed, and top_a was never supported) * Correct the identification string for typical_p in the llama.cpp sampler order block * Add penalties to the llama.cpp sampler order block
| @@ -1284,7 +1284,7 @@ | |||
| 1284 | <input class="neo-range-slider" type="range" id="min_p_textgenerationwebui" name="volume" min="0" max="1" step="0.001"> | 1284 | <input class="neo-range-slider" type="range" id="min_p_textgenerationwebui" name="volume" min="0" max="1" step="0.001"> |
| 1285 | <input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="min_p_textgenerationwebui" id="min_p_counter_textgenerationwebui"> | 1285 | <input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="min_p_textgenerationwebui" id="min_p_counter_textgenerationwebui"> |
| 1286 | </div> | 1286 | </div> |
| 1287 | <div data-tg-type-mode="except" data-tg-type="generic" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> | 1287 | <div data-tg-type-mode="except" data-tg-type="generic,llamacpp" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> |
| 1288 | <small> | 1288 | <small> |
| 1289 | <span data-i18n="Top A">Top A</span> | 1289 | <span data-i18n="Top A">Top A</span> |
| 1290 | <div class="fa-solid fa-circle-info opacity50p" title="Top A sets a threshold for token selection based on the square of the highest token probability. E.g if the Top-A value is 0.2 and the top token's probability is 50%, tokens with probabilities below 5% (0.2 * 0.5^2) are excluded. Set to 0 to disable." data-i18n="[title]Top_A_desc"></div> | 1290 | <div class="fa-solid fa-circle-info opacity50p" title="Top A sets a threshold for token selection based on the square of the highest token probability. E.g if the Top-A value is 0.2 and the top token's probability is 50%, tokens with probabilities below 5% (0.2 * 0.5^2) are excluded. Set to 0 to disable." data-i18n="[title]Top_A_desc"></div> |
| @@ -1292,7 +1292,7 @@ | |||
| 1292 | <input class="neo-range-slider" type="range" id="top_a_textgenerationwebui" name="volume" min="0" max="1" step="0.01"> | 1292 | <input class="neo-range-slider" type="range" id="top_a_textgenerationwebui" name="volume" min="0" max="1" step="0.01"> |
| 1293 | <input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_a_textgenerationwebui" id="top_a_counter_textgenerationwebui"> | 1293 | <input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_a_textgenerationwebui" id="top_a_counter_textgenerationwebui"> |
| 1294 | </div> | 1294 | </div> |
| 1295 | <div data-tg-type-mode="except" data-tg-type="generic" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> | 1295 | <div data-tg-type-mode="except" data-tg-type="generic,llamacpp" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> |
| 1296 | <small> | 1296 | <small> |
| 1297 | <span data-i18n="TFS">TFS</span> | 1297 | <span data-i18n="TFS">TFS</span> |
| 1298 | <div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Tail_Free_Sampling_desc" title="Tail-Free Sampling (TFS) searches for a tail of low-probability tokens in the distribution, by analyzing the rate of change in token probabilities using derivatives. It retains tokens up to a threshold (e.g., 0.3) based on the normalized second derivative. The closer to 0, the more discarded tokens. Set to 1.0 to disable."></div> | 1298 | <div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Tail_Free_Sampling_desc" title="Tail-Free Sampling (TFS) searches for a tail of low-probability tokens in the distribution, by analyzing the rate of change in token probabilities using derivatives. It retains tokens up to a threshold (e.g., 0.3) based on the normalized second derivative. The closer to 0, the more discarded tokens. Set to 1.0 to disable."></div> |
| @@ -1308,13 +1308,21 @@ | |||
| 1308 | <input class="neo-range-slider" type="range" id="epsilon_cutoff_textgenerationwebui" name="volume" min="0" max="9" step="0.01"> | 1308 | <input class="neo-range-slider" type="range" id="epsilon_cutoff_textgenerationwebui" name="volume" min="0" max="9" step="0.01"> |
| 1309 | <input class="neo-range-input" type="number" min="0" max="9" step="0.01" data-for="epsilon_cutoff_textgenerationwebui" id="epsilon_cutoff_counter_textgenerationwebui"> | 1309 | <input class="neo-range-input" type="number" min="0" max="9" step="0.01" data-for="epsilon_cutoff_textgenerationwebui" id="epsilon_cutoff_counter_textgenerationwebui"> |
| 1310 | </div> | 1310 | </div> |
| 1311 | <div data-tg-type="aphrodite,koboldcpp" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> | 1311 | <div data-tg-type="aphrodite,koboldcpp,llamacpp" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> |
| 1312 | <small> | 1312 | <small> |
| 1313 | <span data-i18n="Top nsigma">Top nsigma</span> | 1313 | <span data-i18n="Top nsigma">Top nsigma</span> |
| 1314 | <div class="fa-solid fa-circle-info opacity50p" title="A sampling method that filters logits based on their statistical properties. It keeps tokens within n standard deviations of the maximum logit value, providing a simpler alternative to top-p/top-k sampling while maintaining sampling stability across different temperatures."></div> | 1314 | <div class="fa-solid fa-circle-info opacity50p" title="A sampling method that filters logits based on their statistical properties. It keeps tokens within n standard deviations of the maximum logit value, providing a simpler alternative to top-p/top-k sampling while maintaining sampling stability across different temperatures."></div> |
| 1315 | </small> | 1315 | </small> |
| 1316 | <input class="neo-range-slider" type="range" id="nsigma_textgenerationwebui" name="volume" min="0" max="5" step="0.01"> | 1316 | <input class="neo-range-slider" type="range" id="nsigma_textgenerationwebui" name="volume" min="-0.01" max="5" step="0.01"> |
| 1317 | <input class="neo-range-input" type="number" min="0" max="5" step="0.01" data-for="nsigma_textgenerationwebui" id="nsigma_counter_textgenerationwebui"> | 1317 | <input class="neo-range-input" type="number" min="-0.01" max="5" step="0.01" data-for="nsigma_textgenerationwebui" id="nsigma_counter_textgenerationwebui"> |
| 1318 | </div> | ||
| 1319 | <div data-tg-type="llamacpp" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> | ||
| 1320 | <small> | ||
| 1321 | <span data-i18n="Min Keep">Min Keep</span> | ||
| 1322 | <div class="fa-solid fa-circle-info opacity50p" title="A sampling modifier that ensures that truncation samplers such as top-p, min-p, typical-p, and xtc return at least this many tokens. Set to 0 to disable."></div> | ||
| 1323 | </small> | ||
| 1324 | <input class="neo-range-slider" type="range" id="min_keep_textgenerationwebui" name="volume" min="0" max="50" step="1"> | ||
| 1325 | <input class="neo-range-input" type="number" min="0" max="50" step="1" data-for="min_keep_textgenerationwebui" id="min_keep_counter_textgenerationwebui"> | ||
| 1318 | </div> | 1326 | </div> |
| 1319 | <div data-tg-type="ooba,mancer,aphrodite" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> | 1327 | <div data-tg-type="ooba,mancer,aphrodite" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0"> |
| 1320 | <small> | 1328 | <small> |
| @@ -1769,11 +1777,12 @@ | |||
| 1769 | <div data-name="temperature" draggable="true"><span>Temperature</span><small></small></div> | 1777 | <div data-name="temperature" draggable="true"><span>Temperature</span><small></small></div> |
| 1770 | <div data-name="top_k" draggable="true"><span>Top K</span><small></small></div> | 1778 | <div data-name="top_k" draggable="true"><span>Top K</span><small></small></div> |
| 1771 | <div data-name="top_p" draggable="true"><span>Top P</span><small></small></div> | 1779 | <div data-name="top_p" draggable="true"><span>Top P</span><small></small></div> |
| 1772 | <div data-name="typical_p" draggable="true"><span>Typical P</span><small></small></div> | 1780 | <div data-name="typ_p" draggable="true"><span>Typical P</span><small></small></div> |
| 1773 | <div data-name="tfs_z" draggable="true"><span>Tail Free Sampling</span><small></small></div> | ||
| 1774 | <div data-name="min_p" draggable="true"><span>Min P</span><small></small></div> | 1781 | <div data-name="min_p" draggable="true"><span>Min P</span><small></small></div> |
| 1775 | <div data-name="xtc" draggable="true"><span>Exclude Top Choices</span><small></small></div> | 1782 | <div data-name="xtc" draggable="true"><span>Exclude Top Choices</span><small></small></div> |
| 1776 | <div data-name="dry" draggable="true"><span>DRY</span><small></small></div> | 1783 | <div data-name="dry" draggable="true"><span>DRY</span><small></small></div> |
| 1784 | <div data-name="penalties" draggable="true"><span>Rep/Freq/Pres Penalties</span><small></small></div> | ||
| 1785 | <div data-name="top_n_sigma" draggable="true"><span>Top N-Sigma</span><small></small></div> | ||
| 1777 | </div> | 1786 | </div> |
| 1778 | <div id="llamacpp_samplers_default_order" class="menu_button menu_button_icon"> | 1787 | <div id="llamacpp_samplers_default_order" class="menu_button menu_button_icon"> |
| 1779 | <span data-i18n="Load default order">Load default order</span> | 1788 | <span data-i18n="Load default order">Load default order</span> |
| @@ -56,10 +56,11 @@ const { | |||
| 56 | } = textgen_types; | 56 | } = textgen_types; |
| 57 | 57 | ||
| 58 | const LLAMACPP_DEFAULT_ORDER = [ | 58 | const LLAMACPP_DEFAULT_ORDER = [ |
| 59 | 'penalties', | ||
| 59 | 'dry', | 60 | 'dry', |
| 61 | 'top_n_sigma', | ||
| 60 | 'top_k', | 62 | 'top_k', |
| 61 | 'tfs_z', | 63 | 'typ_p', |
| 62 | 'typical_p', | ||
| 63 | 'top_p', | 64 | 'top_p', |
| 64 | 'min_p', | 65 | 'min_p', |
| 65 | 'xtc', | 66 | 'xtc', |
| @@ -212,6 +213,7 @@ const settings = { | |||
| 212 | xtc_threshold: 0.1, | 213 | xtc_threshold: 0.1, |
| 213 | xtc_probability: 0, | 214 | xtc_probability: 0, |
| 214 | nsigma: 0.0, | 215 | nsigma: 0.0, |
| 216 | min_keep: 0, | ||
| 215 | featherless_model: '', | 217 | featherless_model: '', |
| 216 | generic_model: '', | 218 | generic_model: '', |
| 217 | }; | 219 | }; |
| @@ -294,6 +296,7 @@ export const setting_names = [ | |||
| 294 | 'xtc_threshold', | 296 | 'xtc_threshold', |
| 295 | 'xtc_probability', | 297 | 'xtc_probability', |
| 296 | 'nsigma', | 298 | 'nsigma', |
| 299 | 'min_keep', | ||
| 297 | 'generic_model', | 300 | 'generic_model', |
| 298 | ]; | 301 | ]; |
| 299 | 302 | ||
| @@ -803,7 +806,8 @@ jQuery(function () { | |||
| 803 | 'dry_penalty_last_n_textgenerationwebui': 0, | 806 | 'dry_penalty_last_n_textgenerationwebui': 0, |
| 804 | 'xtc_threshold_textgenerationwebui': 0.1, | 807 | 'xtc_threshold_textgenerationwebui': 0.1, |
| 805 | 'xtc_probability_textgenerationwebui': 0, | 808 | 'xtc_probability_textgenerationwebui': 0, |
| 806 | 'nsigma_textgenerationwebui': 0, | 809 | 'nsigma_textgenerationwebui': [LLAMACPP].includes(settings.type) ? -0.01 : 0, |
| 810 | 'min_keep_textgenerationwebui': 0, | ||
| 807 | }; | 811 | }; |
| 808 | 812 | ||
| 809 | for (const [id, value] of Object.entries(inputs)) { | 813 | for (const [id, value] of Object.entries(inputs)) { |
| @@ -1332,6 +1336,8 @@ export async function getTextGenGenerationData(finalPrompt, maxTokens, isImperso | |||
| 1332 | 'xtc_threshold': settings.xtc_threshold, | 1336 | 'xtc_threshold': settings.xtc_threshold, |
| 1333 | 'xtc_probability': settings.xtc_probability, | 1337 | 'xtc_probability': settings.xtc_probability, |
| 1334 | 'nsigma': settings.nsigma, | 1338 | 'nsigma': settings.nsigma, |
| 1339 | 'top_n_sigma': settings.nsigma, | ||
| 1340 | 'min_keep': settings.min_keep, | ||
| 1335 | }; | 1341 | }; |
| 1336 | const nonAphroditeParams = { | 1342 | const nonAphroditeParams = { |
| 1337 | 'rep_pen': settings.rep_pen, | 1343 | 'rep_pen': settings.rep_pen, |