Text Completion: Add global banned strings list

412d638e9e7335b9e1416cf512c3af5bb6469959

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: f101dd571fc283f009e72c38619f4aa2f09b3f50
3 files changed, +15 -2Showing whitespace changes
public/index.html+9 -0
@@ -1625,6 +1625,15 @@
1625 <span data-i18n="Banned Tokens">Banned Tokens/Strings</span>1625 <span data-i18n="Banned Tokens">Banned Tokens/Strings</span>
1626 <div class="margin5 fa-solid fa-circle-info opacity50p " data-i18n="[title]LLaMA / Mistral / Yi models only" title="Enter sequences you don't want to appear in the output.&#13;Unquoted text will be tokenized in the back end and banned as tokens.&#13;[token ids] will be banned as-is.&#13;Most tokens have a leading space. Use token counter (with the correct tokenizer selected first!) if you are unsure.&#13;Enclose text in double quotes to ban the entire string as a set.&#13;Quoted Strings and [Token ids] must be on their own line."></div>1626 <div class="margin5 fa-solid fa-circle-info opacity50p " data-i18n="[title]LLaMA / Mistral / Yi models only" title="Enter sequences you don't want to appear in the output.&#13;Unquoted text will be tokenized in the back end and banned as tokens.&#13;[token ids] will be banned as-is.&#13;Most tokens have a leading space. Use token counter (with the correct tokenizer selected first!) if you are unsure.&#13;Enclose text in double quotes to ban the entire string as a set.&#13;Quoted Strings and [Token ids] must be on their own line."></div>
1627 </h4>1627 </h4>
1628 <div class="textAlignCenter">
1629 <small data-i18n="Global list">Global list</small>
1630 </div>
1631 <div class="wide100p marginBot10">
1632 <textarea id="global_banned_tokens_textgenerationwebui" class="text_pole textarea_compact" name="global_banned_tokens_textgenerationwebui" rows="3" data-i18n="[placeholder]Example: some text [42, 69, 1337]" placeholder='some text as tokens&#10;[420, 69, 1337]&#10;"Some verbatim string"'></textarea>
1633 </div>
1634 <div class="textAlignCenter">
1635 <small data-i18n="Preset-specific list">Preset-specific list</small>
1636 </div>
1628 <div class="wide100p">1637 <div class="wide100p">
1629 <textarea id="banned_tokens_textgenerationwebui" class="text_pole textarea_compact" name="banned_tokens_textgenerationwebui" rows="3" data-i18n="[placeholder]Example: some text [42, 69, 1337]" placeholder='some text as tokens&#10;[420, 69, 1337]&#10;"Some verbatim string"'></textarea>1638 <textarea id="banned_tokens_textgenerationwebui" class="text_pole textarea_compact" name="banned_tokens_textgenerationwebui" rows="3" data-i18n="[placeholder]Example: some text [42, 69, 1337]" placeholder='some text as tokens&#10;[420, 69, 1337]&#10;"Some verbatim string"'></textarea>
1630 </div>1639 </div>
public/scripts/preset-manager.js+1 -0
@@ -587,6 +587,7 @@ class PresetManager {
587 'derived',587 'derived',
588 'generic_model',588 'generic_model',
589 'include_reasoning',589 'include_reasoning',
590 'global_banned_tokens',
590 ];591 ];
591 const settings = Object.assign({}, getSettingsByApiId(this.apiId));592 const settings = Object.assign({}, getSettingsByApiId(this.apiId));
592593
public/scripts/textgen-settings.js+5 -2
@@ -182,6 +182,7 @@ const settings = {
182 grammar_string: '',182 grammar_string: '',
183 json_schema: {},183 json_schema: {},
184 banned_tokens: '',184 banned_tokens: '',
185 global_banned_tokens: '',
185 sampler_priority: OOBA_DEFAULT_ORDER,186 sampler_priority: OOBA_DEFAULT_ORDER,
186 samplers: LLAMACPP_DEFAULT_ORDER,187 samplers: LLAMACPP_DEFAULT_ORDER,
187 samplers_priorities: APHRODITE_DEFAULT_ORDER,188 samplers_priorities: APHRODITE_DEFAULT_ORDER,
@@ -274,6 +275,7 @@ export const setting_names = [
274 'grammar_string',275 'grammar_string',
275 'json_schema',276 'json_schema',
276 'banned_tokens',277 'banned_tokens',
278 'global_banned_tokens',
277 'ignore_eos_token',279 'ignore_eos_token',
278 'spaces_between_special_tokens',280 'spaces_between_special_tokens',
279 'speculative_ngram',281 'speculative_ngram',
@@ -404,8 +406,9 @@ function getCustomTokenBans() {
404 const tokenizer = getTokenizerForTokenIds();406 const tokenizer = getTokenizerForTokenIds();
405 const banned_tokens = [];407 const banned_tokens = [];
406 const banned_strings = [];408 const banned_strings = [];
407 const sequences = settings.banned_tokens409 const sequences = []
408 .split('\n')410 .concat(settings.banned_tokens.split('\n'))
411 .concat(settings.global_banned_tokens.split('\n'))
409 .concat(textgenerationwebui_banned_in_macros)412 .concat(textgenerationwebui_banned_in_macros)
410 .filter(x => x.length > 0)413 .filter(x => x.length > 0)
411 .filter(onlyUnique);414 .filter(onlyUnique);