"N" support for llama.cpp (#4869) * llama.cpp supports 'n' now * Fix response parsing --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

b5e20f2ff98708b13961d09130b63eda839c191b

Beinsezii <39478211+Beinsezii@users.noreply.github.com>

Signed
Parents: 9aff57c9c43d2b006709cc79f5e311659048abdd
4 files changed, +40 -11Showing whitespace changes
public/index.html+1 -1
@@ -1249,7 +1249,7 @@
12491249 <div class="fa-solid fa-circle-info opacity50p" title="Customize displayed samplers or add custom samplers." data-i18n="[title]Customize displayed samplers or add custom samplers."></div>
12501250 </small>
12511251 </div>
12521252 <div data-tg-type="mancer, vllm, aphrodite, tabby, infermaticai, llamacpp" data-tg-samplers="n" class="flex-container flexFlowColumn alignitemscenter flexBasis100p flexGrow flexShrink gap0">
12531253 <small data-i18n="Multiple swipes per generation">Multiple swipes per generation</small>
12541254 <input type="number" id="n_textgenerationwebui" class="text_pole textAlignCenter" min="1" value="1" step="1" />
12551255 </div>
public/script.js+24 -8
@@ -5872,7 +5872,7 @@ export function extractMessageFromData(data, activeApi = null) {
58725872 case 'koboldhorde':
58735873 return data.text;
58745874 case 'textgenerationwebui':
58755875 return data.choices?.[0]?.text ?? data.choices?.[0]?.message?.content ?? data.content ?? data.response ?? data[0]?.content ?? '';
58765876 case 'novel':
58775877 return data.output;
58785878 case 'openai':
@@ -5959,6 +5959,22 @@ function extractMultiSwipes(data, type) {
59595959 return swipes;
59605960 }
59615961
5962+ if (main_api === 'textgenerationwebui' && textgen_settings.type === textgen_types.LLAMACPP) {
5963+ if (!Array.isArray(data)) {
5964+ return swipes;
5965+ }
5966+
5967+ const multiSwipeCount = data.length - 1;
5968+ if (multiSwipeCount <= 0) {
5969+ return swipes;
5970+ }
5971+
5972+ for (let i = 1; i < data.length; i++) {
5973+ const text = data?.[i]?.content ?? '';
5974+ swipes.push(text);
5975+ }
5976+ }
5977+
59625978 if (main_api === 'openai' || (main_api === 'textgenerationwebui' && [textgen_types.MANCER, textgen_types.VLLM, textgen_types.APHRODITE, textgen_types.TABBY, textgen_types.INFERMATICAI].includes(textgen_settings.type))) {
59635979 if (!Array.isArray(data.choices)) {
59645980 return swipes;
@@ -5972,18 +5988,18 @@ function extractMultiSwipes(data, type) {
59725988
59735989 for (let i = 1; i < data.choices.length; i++) {
59745990 const text = data?.choices[i]?.message?.content ?? data?.choices[i]?.text ?? '';
5975- const cleanedText = cleanUpMessage({
5991+ swipes.push(text);
5992+ }
5993+ }
5994+
5995+ const cleanedSwipes = swipes.map(text => cleanUpMessage({
59765996 getMessage: text,
59775997 isImpersonate: false,
59785998 isContinue: false,
59795999 displayIncompleteSentences: false,
59806000 }));
5981-
5982- swipes.push(cleanedText);
5983- }
5984- }
59856001
59866002 return swipescleanedSwipes;
59876003}
59886004
59896005/**
public/scripts/sse-stream.js+10 -1
@@ -1,6 +1,9 @@
11import { power_user } from './power-user.js';
22import { delay } from './utils.js';
33
4+// Symbol for not primary swipe error
5+const NOT_PRIMARY = Symbol('not_primary_swipe');
6+
47/**
58 * A stream which handles Server-Sent Events from a binary ReadableStream like you get from the fetch API.
69 */
@@ -198,6 +201,10 @@ async function* parseStreamData(json) {
198201 }
199202 // llama.cpp?
200203 else if (typeof json.content === 'string' && json.content.length > 0 && json.object !== 'chat.completion.chunk') {
204+ const isNotPrimary = json?.index > 0;
205+ if (isNotPrimary) {
206+ throw new Error('Not a primary swipe', { cause: NOT_PRIMARY });
207+ }
201208 for (let i = 0; i < json.content.length; i++) {
202209 const str = json.content[i];
203210 yield {
@@ -211,7 +218,7 @@ async function* parseStreamData(json) {
211218 else if (Array.isArray(json.choices)) {
212219 const isNotPrimary = json?.choices?.[0]?.index > 0;
213220 if (isNotPrimary || json.choices.length === 0) {
214221 throw new Error('Not a primary swipe', { cause: NOT_PRIMARY });
215222 }
216223
217224 if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) {
@@ -357,7 +364,9 @@ export class SmoothEventSourceStream extends EventSourceStream {
357364 lastStr = parsed.chunk;
358365 }
359366 } catch (error) {
367+ if (error instanceof Error && error.cause !== NOT_PRIMARY) {
360368 console.debug('Smooth Streaming parsing error', error);
369+ }
361370 controller.enqueue(event);
362371 }
363372 },
public/scripts/textgen-settings.js+5 -1
@@ -1284,6 +1284,10 @@ export async function generateTextGenWithStreaming(generate_data, signal) {
12841284 if (data?.choices?.[0]?.index > 0) {
12851285 const swipeIndex = data.choices[0].index - 1;
12861286 swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.choices[0].text;
1287+ } else if (data?.index > 0) {
1288+ // llama.cpp streaming swipe
1289+ const swipeIndex = data.index - 1;
1290+ swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.content;
12871291 } else {
12881292 const newText = data?.choices?.[0]?.text || data?.content || '';
12891293 text += newText;
@@ -1724,7 +1728,7 @@ export function createTextGenGenerationData(settings, model, finalPrompt = null,
17241728 params.dry_sequence_breakers = params.parseSequenceBreakers();
17251729 }
17261730
17271731 if (settings.type === TABBY || settings.type === LLAMACPP) {
17281732 params.n = canMultiSwipe ? settings.n : 1;
17291733 }
17301734