Merge pull request #3980 from SillyTavern/fix-lcpp-caption Fix llama.cpp captioning

b2ed69aac2b7c334c0b7d6d49657217188b0d412

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: c6a64d85265b417683a8449766635222c6980524 aef005007f3b7810a90abb7d47106bb349344bfb
3 files changed, +8 -50Showing whitespace changes
public/scripts/extensions/shared.js+1 -3
@@ -22,7 +22,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
2222
23 throwIfInvalidModel(useReverseProxy);23 throwIfInvalidModel(useReverseProxy);
2424
25 const noPrefix = ['ollama', 'llamacpp'].includes(extension_settings.caption.multimodal_api);25 const noPrefix = ['ollama'].includes(extension_settings.caption.multimodal_api);
2626
27 if (noPrefix && base64Img.startsWith('data:image/')) {27 if (noPrefix && base64Img.startsWith('data:image/')) {
28 base64Img = base64Img.split(',')[1];28 base64Img = base64Img.split(',')[1];
@@ -97,8 +97,6 @@ export async function getMultimodalCaption(base64Img, prompt) {
97 return '/api/google/caption-image';97 return '/api/google/caption-image';
98 case 'anthropic':98 case 'anthropic':
99 return '/api/anthropic/caption-image';99 return '/api/anthropic/caption-image';
100 case 'llamacpp':
101 return '/api/backends/text-completions/llamacpp/caption-image';
102 case 'ollama':100 case 'ollama':
103 return '/api/backends/text-completions/ollama/caption-image';101 return '/api/backends/text-completions/ollama/caption-image';
104 default:102 default:
src/endpoints/backends/text-completions.js+0 -45
@@ -502,51 +502,6 @@ ollama.post('/caption-image', async function (request, response) {
502502
503const llamacpp = express.Router();503const llamacpp = express.Router();
504504
505llamacpp.post('/caption-image', async function (request, response) {
506 try {
507 if (!request.body.server_url) {
508 return response.sendStatus(400);
509 }
510
511 console.debug('LlamaCpp caption request:', request.body);
512 const baseUrl = trimV1(request.body.server_url);
513
514 const fetchResponse = await fetch(`${baseUrl}/completion`, {
515 method: 'POST',
516 headers: { 'Content-Type': 'application/json' },
517 body: JSON.stringify({
518 prompt: `USER:[img-1]${String(request.body.prompt).trim()}\nASSISTANT:`,
519 image_data: [{ data: request.body.image, id: 1 }],
520 temperature: 0.1,
521 stream: false,
522 stop: ['USER:', '</s>'],
523 }),
524 });
525
526 if (!fetchResponse.ok) {
527 console.error('LlamaCpp caption error:', fetchResponse.status, fetchResponse.statusText);
528 return response.status(500).send({ error: true });
529 }
530
531 /** @type {any} */
532 const data = await fetchResponse.json();
533 console.debug('LlamaCpp caption response:', data);
534
535 const caption = data?.content || '';
536
537 if (!caption) {
538 console.error('LlamaCpp caption is empty.');
539 return response.status(500).send({ error: true });
540 }
541
542 return response.send({ caption });
543
544 } catch (error) {
545 console.error(error);
546 return response.sendStatus(500);
547 }
548});
549
550llamacpp.post('/props', async function (request, response) {505llamacpp.post('/props', async function (request, response) {
551 try {506 try {
552 if (!request.body.server_url) {507 if (!request.body.server_url) {
src/endpoints/openai.js+7 -2
@@ -45,6 +45,10 @@ router.post('/caption-image', async (request, response) => {
45 key = readSecret(request.user.directories, SECRET_KEYS.KOBOLDCPP);45 key = readSecret(request.user.directories, SECRET_KEYS.KOBOLDCPP);
46 }46 }
4747
48 if (request.body.api === 'llamacpp') {
49 key = readSecret(request.user.directories, SECRET_KEYS.LLAMACPP);
50 }
51
48 if (request.body.api === 'vllm') {52 if (request.body.api === 'vllm') {
49 key = readSecret(request.user.directories, SECRET_KEYS.VLLM);53 key = readSecret(request.user.directories, SECRET_KEYS.VLLM);
50 }54 }
@@ -69,7 +73,8 @@ router.post('/caption-image', async (request, response) => {
69 key = readSecret(request.user.directories, SECRET_KEYS.XAI);73 key = readSecret(request.user.directories, SECRET_KEYS.XAI);
70 }74 }
7175
72 if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) {76 const noKeyTypes = ['custom', 'ooba', 'koboldcpp', 'vllm', 'llamacpp'];
77 if (!key && !request.body.reverse_proxy && !noKeyTypes.includes(request.body.api)) {
73 console.warn('No key found for API', request.body.api);78 console.warn('No key found for API', request.body.api);
74 return response.sendStatus(400);79 return response.sendStatus(400);
75 }80 }
@@ -156,7 +161,7 @@ router.post('/caption-image', async (request, response) => {
156 });161 });
157 }162 }
158163
159 if (request.body.api === 'koboldcpp' || request.body.api === 'vllm') {164 if (['koboldcpp', 'vllm', 'llamacpp'].includes(request.body.api)) {
160 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;165 apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
161 }166 }
162167