Text Completion: Add global banned strings list

412d638e9e7335b9e1416cf512c3af5bb6469959

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: f101dd571fc283f009e72c38619f4aa2f09b3f50
3 files changed, +15 -2Ignore whitespace
public/index.html+9 -0
@@ -1625,6 +1625,15 @@
16251625 <span data-i18n="Banned Tokens">Banned Tokens/Strings</span>
16261626 <div class="margin5 fa-solid fa-circle-info opacity50p " data-i18n="[title]LLaMA / Mistral / Yi models only" title="Enter sequences you don't want to appear in the output.&#13;Unquoted text will be tokenized in the back end and banned as tokens.&#13;[token ids] will be banned as-is.&#13;Most tokens have a leading space. Use token counter (with the correct tokenizer selected first!) if you are unsure.&#13;Enclose text in double quotes to ban the entire string as a set.&#13;Quoted Strings and [Token ids] must be on their own line."></div>
16271627 </h4>
1628+ <div class="textAlignCenter">
1629+ <small data-i18n="Global list">Global list</small>
1630+ </div>
1631+ <div class="wide100p marginBot10">
1632+ <textarea id="global_banned_tokens_textgenerationwebui" class="text_pole textarea_compact" name="global_banned_tokens_textgenerationwebui" rows="3" data-i18n="[placeholder]Example: some text [42, 69, 1337]" placeholder='some text as tokens&#10;[420, 69, 1337]&#10;"Some verbatim string"'></textarea>
1633+ </div>
1634+ <div class="textAlignCenter">
1635+ <small data-i18n="Preset-specific list">Preset-specific list</small>
1636+ </div>
16281637 <div class="wide100p">
16291638 <textarea id="banned_tokens_textgenerationwebui" class="text_pole textarea_compact" name="banned_tokens_textgenerationwebui" rows="3" data-i18n="[placeholder]Example: some text [42, 69, 1337]" placeholder='some text as tokens&#10;[420, 69, 1337]&#10;"Some verbatim string"'></textarea>
16301639 </div>
public/scripts/preset-manager.js+1 -0
@@ -587,6 +587,7 @@ class PresetManager {
587587 'derived',
588588 'generic_model',
589589 'include_reasoning',
590+ 'global_banned_tokens',
590591 ];
591592 const settings = Object.assign({}, getSettingsByApiId(this.apiId));
592593
public/scripts/textgen-settings.js+5 -2
@@ -182,6 +182,7 @@ const settings = {
182182 grammar_string: '',
183183 json_schema: {},
184184 banned_tokens: '',
185+ global_banned_tokens: '',
185186 sampler_priority: OOBA_DEFAULT_ORDER,
186187 samplers: LLAMACPP_DEFAULT_ORDER,
187188 samplers_priorities: APHRODITE_DEFAULT_ORDER,
@@ -274,6 +275,7 @@ export const setting_names = [
274275 'grammar_string',
275276 'json_schema',
276277 'banned_tokens',
278+ 'global_banned_tokens',
277279 'ignore_eos_token',
278280 'spaces_between_special_tokens',
279281 'speculative_ngram',
@@ -404,8 +406,9 @@ function getCustomTokenBans() {
404406 const tokenizer = getTokenizerForTokenIds();
405407 const banned_tokens = [];
406408 const banned_strings = [];
407409 const sequences = settings.banned_tokens[]
408410 .concat(settings.banned_tokens.split('\n'))
411+ .concat(settings.global_banned_tokens.split('\n'))
409412 .concat(textgenerationwebui_banned_in_macros)
410413 .filter(x => x.length > 0)
411414 .filter(onlyUnique);