Stable diffusion.cpp server support (#5074) * feat: Add stable-diffusion.cpp server endpoints * feat: Add stable-diffusion.cpp server settings * feat: Add stable-diffusion.cpp server * Add debug log for sdcpp payload * feat: add loadSdcppSamplers and loadSdcppSchedulers for sdcpp backend * feat: add sdcpp to sampler and scheduler source lists --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

12452491cbf8b64f8f40ac93e2dd0dfb9054c2b4

Jay4242 <96741732+Jay4242@users.noreply.github.com>

Signed
Parents: 10e8e01a554d1f4fc6b0a358e03b65ab0e84bc76
3 files changed, +197 -4Ignore whitespace
public/scripts/extensions/stable-diffusion/index.js+111 -0
@@ -79,6 +79,7 @@ const sources = {
7979 extras: 'extras',
8080 horde: 'horde',
8181 auto: 'auto',
82+ sdcpp: 'sdcpp',
8283 novel: 'novel',
8384 vlad: 'vlad',
8485 openai: 'openai',
@@ -290,6 +291,9 @@ const defaultSettings = {
290291 auto_url: 'http://localhost:7860',
291292 auto_auth: '',
292293
294+ // stable-diffusion.cpp settings
295+ sdcpp_url: 'http://127.0.0.1:1234',
296+
293297 vlad_url: 'http://localhost:7860',
294298 vlad_auth: '',
295299
@@ -528,6 +532,7 @@ async function loadSettings() {
528532 $('#sd_multimodal_captioning').prop('checked', extension_settings.sd.multimodal_captioning);
529533 $('#sd_auto_url').val(extension_settings.sd.auto_url);
530534 $('#sd_auto_auth').val(extension_settings.sd.auto_auth);
535+ $('#sd_sdcpp_url').val(extension_settings.sd.sdcpp_url);
531536 $('#sd_vlad_url').val(extension_settings.sd.vlad_url);
532537 $('#sd_vlad_auth').val(extension_settings.sd.vlad_auth);
533538 $('#sd_drawthings_url').val(extension_settings.sd.drawthings_url);
@@ -1156,6 +1161,11 @@ function onAutoAuthInput() {
11561161 saveSettingsDebounced();
11571162}
11581163
1164+function onSdcppUrlInput() {
1165+ extension_settings.sd.sdcpp_url = $('#sd_sdcpp_url').val();
1166+ saveSettingsDebounced();
1167+}
1168+
11591169function onVladUrlInput() {
11601170 extension_settings.sd.vlad_url = $('#sd_vlad_url').val();
11611171 saveSettingsDebounced();
@@ -1264,6 +1274,29 @@ async function validateAutoUrl() {
12641274 }
12651275}
12661276
1277+async function validateSdcppUrl() {
1278+ try {
1279+ if (!extension_settings.sd.sdcpp_url) {
1280+ throw new Error('URL is not set.');
1281+ }
1282+
1283+ const result = await fetch('/api/sd/sdcpp/ping', {
1284+ method: 'POST',
1285+ headers: getRequestHeaders(),
1286+ body: JSON.stringify({ url: extension_settings.sd.sdcpp_url }),
1287+ });
1288+
1289+ if (!result.ok) {
1290+ throw new Error('stable-diffusion.cpp server returned an error.');
1291+ }
1292+
1293+ await loadSettingOptions();
1294+ toastr.success('stable-diffusion.cpp server connected.');
1295+ } catch (error) {
1296+ toastr.error(`Could not validate stable-diffusion.cpp server: ${error.message}`);
1297+ }
1298+}
1299+
12671300async function validateDrawthingsUrl() {
12681301 try {
12691302 if (!extension_settings.sd.drawthings_url) {
@@ -1557,6 +1590,9 @@ async function loadSamplers() {
15571590 case sources.auto:
15581591 samplers = await loadAutoSamplers();
15591592 break;
1593+ case sources.sdcpp:
1594+ samplers = await loadSdcppSamplers();
1595+ break;
15601596 case sources.drawthings:
15611597 samplers = await loadDrawthingsSamplers();
15621598 break;
@@ -1682,6 +1718,11 @@ async function loadAutoSamplers() {
16821718 }
16831719}
16841720
1721+async function loadSdcppSamplers() {
1722+ // The sdcpp server does not provide an API for samplers, so we return the known list.
1723+ return ['euler', 'euler_a', 'heun', 'dpm2', 'dpm++2s_a', 'dpm++2m', 'dpm++2mv2', 'ipndm', 'ipndm_v', 'lcm', 'ddim_trailing', 'tcd'];
1724+}
1725+
16851726async function loadDrawthingsSamplers() {
16861727 // The app developer doesn't provide an API to get these yet
16871728 return [
@@ -1771,6 +1812,9 @@ async function loadModels() {
17711812 case sources.auto:
17721813 models = await loadAutoModels();
17731814 break;
1815+ case sources.sdcpp:
1816+ models = [{ value: '', text: 'N/A' }];
1817+ break;
17741818 case sources.drawthings:
17751819 models = await loadDrawthingsModels();
17761820 break;
@@ -2379,6 +2423,9 @@ async function loadSchedulers() {
23792423 case sources.auto:
23802424 schedulers = await getAutoRemoteSchedulers();
23812425 break;
2426+ case sources.sdcpp:
2427+ schedulers = await loadSdcppSchedulers();
2428+ break;
23822429 case sources.novel:
23832430 schedulers = loadNovelSchedulers();
23842431 break;
@@ -2477,6 +2524,11 @@ async function loadComfySchedulers() {
24772524 }
24782525}
24792526
2527+async function loadSdcppSchedulers() {
2528+ // The sdcpp server does not provide an API for schedulers, so we return the known list.
2529+ return ['discrete', 'karras', 'exponential', 'ays', 'gits', 'smoothstep', 'sgm_uniform', 'simple', 'kl_optimal', 'lcm'];
2530+}
2531+
24802532async function loadVaes() {
24812533 $('#sd_vae').empty();
24822534 let vaes = [];
@@ -2491,6 +2543,9 @@ async function loadVaes() {
24912543 case sources.auto:
24922544 vaes = await loadAutoVaes();
24932545 break;
2546+ case sources.sdcpp:
2547+ vaes = ['N/A'];
2548+ break;
24942549 case sources.novel:
24952550 vaes = ['N/A'];
24962551 break;
@@ -3166,6 +3221,9 @@ async function sendGenerationRequest(generationType, prompt, additionalNegativeP
31663221 case sources.auto:
31673222 result = await generateAutoImage(prefixedPrompt, negativePrompt, signal);
31683223 break;
3224+ case sources.sdcpp:
3225+ result = await generateSdcppImage(prefixedPrompt, negativePrompt, signal);
3226+ break;
31693227 case sources.novel:
31703228 result = await generateNovelImage(prefixedPrompt, negativePrompt, signal);
31713229 break;
@@ -3654,6 +3712,55 @@ async function generateAutoImage(prompt, negativePrompt, signal) {
36543712}
36553713
36563714/**
3715+ * Generates an image using stable-diffusion.cpp server API.
3716+ *
3717+ * @param {string} prompt - The main instruction used to guide the image generation.
3718+ * @param {string} negativePrompt - The instruction used to restrict the image generation.
3719+ * @param {AbortSignal} signal - An AbortSignal object that can be used to cancel the request.
3720+ * @returns {Promise<{format: string, data: string}>} - A promise that resolves when the image generation and processing are complete.
3721+ */
3722+async function generateSdcppImage(prompt, negativePrompt, signal) {
3723+ const payload = {
3724+ url: extension_settings.sd.sdcpp_url,
3725+ prompt: prompt,
3726+ negative_prompt: negativePrompt,
3727+ steps: extension_settings.sd.steps,
3728+ cfg_scale: extension_settings.sd.scale,
3729+ width: extension_settings.sd.width,
3730+ height: extension_settings.sd.height,
3731+ batch_size: 1,
3732+ seed: extension_settings.sd.seed >= 0 ? extension_settings.sd.seed : undefined,
3733+ };
3734+
3735+ if (extension_settings.sd.sampler && extension_settings.sd.sampler !== 'N/A') {
3736+ payload.sampler_name = extension_settings.sd.sampler;
3737+ }
3738+
3739+ if (extension_settings.sd.scheduler && extension_settings.sd.scheduler !== 'N/A') {
3740+ payload.scheduler = extension_settings.sd.scheduler;
3741+ }
3742+
3743+ if (Number.isFinite(extension_settings.sd.clip_skip)) {
3744+ payload.clip_skip = extension_settings.sd.clip_skip;
3745+ }
3746+
3747+ const result = await fetch('/api/sd/sdcpp/generate', {
3748+ method: 'POST',
3749+ headers: getRequestHeaders(),
3750+ signal: signal,
3751+ body: JSON.stringify(payload),
3752+ });
3753+
3754+ if (result.ok) {
3755+ const data = await result.json();
3756+ return { format: 'png', data: data.images?.[0] };
3757+ } else {
3758+ const text = await result.text();
3759+ throw new Error(text);
3760+ }
3761+}
3762+
3763+/**
36573764 * Generates an image in Drawthings API using the provided prompt and configuration settings.
36583765 *
36593766 * @param {string} prompt - The main instruction used to guide the image generation.
@@ -4736,6 +4843,8 @@ function isValidState() {
47364843 return true;
47374844 case sources.auto:
47384845 return !!extension_settings.sd.auto_url;
4846+ case sources.sdcpp:
4847+ return !!extension_settings.sd.sdcpp_url;
47394848 case sources.drawthings:
47404849 return !!extension_settings.sd.drawthings_url;
47414850 case sources.vlad:
@@ -5418,6 +5527,8 @@ jQuery(async () => {
54185527 $('#sd_auto_validate').on('click', validateAutoUrl);
54195528 $('#sd_auto_url').on('input', onAutoUrlInput);
54205529 $('#sd_auto_auth').on('input', onAutoAuthInput);
5530+ $('#sd_sdcpp_validate').on('click', validateSdcppUrl);
5531+ $('#sd_sdcpp_url').on('input', onSdcppUrlInput);
54215532 $('#sd_drawthings_validate').on('click', validateDrawthingsUrl);
54225533 $('#sd_drawthings_url').on('input', onDrawthingsUrlInput);
54235534 $('#sd_drawthings_auth').on('input', onDrawthingsAuthInput);
public/scripts/extensions/stable-diffusion/settings.html+18 -4
@@ -55,6 +55,7 @@
5555 <option value="vlad">SD.Next (vladmandic)</option>
5656 <option value="stability">Stability AI</option>
5757 <option value="auto">Stable Diffusion Web UI (AUTOMATIC1111)</option>
58+ <option value="sdcpp">stable-diffusion.cpp server</option>
5859 <option value="horde">Stable Horde</option>
5960 <option value="togetherai">TogetherAI</option>
6061 <option value="xai">xAI (Grok)</option>
@@ -76,6 +77,19 @@
7677 <!-- (Original Text)<b>Important:</b> run SD Web UI with the <tt>--api</tt> flag! The server must be accessible from the SillyTavern host machine. -->
7778 <i><b data-i18n="Important:">Important:</b></i><i data-i18n="sd_auto_auth_warning_1"> run SD Web UI with the </i><i><tt>--api</tt></i><i data-i18n="sd_auto_auth_warning_2"> flag! The server must be accessible from the SillyTavern host machine.</i>
7879 </div>
80+ <div data-sd-source="sdcpp">
81+ <label for="sd_sdcpp_url">stable-diffusion.cpp URL</label>
82+ <div class="flex-container flexnowrap">
83+ <input id="sd_sdcpp_url" type="text" class="text_pole" placeholder="Example: {{sdcpp_url}}" value="{{sdcpp_url}}" />
84+ <div id="sd_sdcpp_validate" class="menu_button menu_button_icon">
85+ <i class="fa-solid fa-check"></i>
86+ <span data-i18n="Connect">
87+ Connect
88+ </span>
89+ </div>
90+ </div>
91+ <i data-i18n="The server must be accessible from the SillyTavern host machine.">The server must be accessible from the SillyTavern host machine.</i>
92+ </div>
7993 <div data-sd-source="drawthings">
8094 <label for="sd_drawthings_url">DrawThings API URL</label>
8195 <div class="flex-container flexnowrap">
@@ -395,12 +409,12 @@
395409 </div>
396410
397411 <div class="flex-container">
398412 <div class="flex1" data-sd-source="extras,horde,auto,drawthings,novel,vlad,comfy,sdcpp">
399413 <label for="sd_sampler" data-i18n="Sampling method">Sampling method</label>
400414 <select id="sd_sampler"></select>
401415 </div>
402416
403417 <div class="flex1" data-sd-source="comfy,auto,novel,sdcpp">
404418 <label for="sd_scheduler" data-i18n="Scheduler">Scheduler</label>
405419 <select id="sd_scheduler"></select>
406420 </div>
@@ -483,7 +497,7 @@
483497 <input class="neo-range-input" type="number" id="sd_hr_second_pass_steps_value" data-for="sd_hr_second_pass_steps" max="{{hr_second_pass_steps_max}}" step="{{hr_second_pass_steps_step}}" value="{{hr_second_pass_steps}}" >
484498 </div>
485499
486500 <div class="alignitemscenter flex-container flexFlowColumn flexGrow flexShrink gap0 flexBasis48p" data-sd-source="auto,vlad,comfy,horde,drawthings,extras,sdcpp">
487501 <small>
488502 <span data-i18n="CLIP Skip">CLIP Skip</span>
489503 </small>
@@ -537,7 +551,7 @@
537551 </label>
538552 </div>
539553
540554 <div data-sd-source="novel,togetherai,pollinations,comfy,drawthings,vlad,auto,horde,extras,stability,bfl,sdcpp" class="marginTop5">
541555 <label for="sd_seed">
542556 <span data-i18n="Seed">Seed</span>
543557 <small data-i18n="(-1 for random)">(-1 for random)</small>
src/endpoints/stable-diffusion.js+68 -0
@@ -801,6 +801,73 @@ together.post('/generate', async (request, response) => {
801801 }
802802});
803803
804+const sdcpp = express.Router();
805+
806+sdcpp.post('/ping', async (request, response) => {
807+ try {
808+ const url = new URL(request.body.url);
809+ url.pathname = '/v1/images/generations';
810+
811+ const result = await fetch(url, { method: 'OPTIONS' });
812+ if (!result.ok) {
813+ throw new Error('stable-diffusion.cpp server returned an error.');
814+ }
815+
816+ return response.sendStatus(200);
817+ } catch (error) {
818+ console.error(error);
819+ return response.sendStatus(500);
820+ }
821+});
822+
823+sdcpp.post('/generate', async (request, response) => {
824+ try {
825+ const url = new URL(request.body.url);
826+ url.pathname = '/sdapi/v1/txt2img';
827+
828+ const payload = {
829+ prompt: request.body.prompt,
830+ negative_prompt: request.body.negative_prompt,
831+ width: request.body.width,
832+ height: request.body.height,
833+ steps: request.body.steps,
834+ cfg_scale: request.body.cfg_scale,
835+ seed: request.body.seed,
836+ batch_size: request.body.batch_size,
837+ sampler_name: request.body.sampler_name,
838+ scheduler: request.body.scheduler,
839+ clip_skip: request.body.clip_skip,
840+ };
841+
842+ for (const [key, value] of Object.entries(payload)) {
843+ if (value === undefined || value === null || value === '') {
844+ delete payload[key];
845+ }
846+ }
847+
848+ console.debug('stable-diffusion.cpp request:', payload);
849+
850+ const result = await fetch(url, {
851+ method: 'POST',
852+ body: JSON.stringify(payload),
853+ headers: {
854+ 'Content-Type': 'application/json',
855+ },
856+ });
857+
858+ if (!result.ok) {
859+ const text = await result.text();
860+ throw new Error('stable-diffusion.cpp server returned an error.', { cause: text });
861+ }
862+
863+ const data = await result.json();
864+ return response.send(data);
865+ } catch (error) {
866+ console.error(error);
867+ return response.sendStatus(500);
868+ }
869+});
870+
804871const drawthings = express.Router();
805872
806873drawthings.post('/ping', async (request, response) => {
@@ -1917,6 +1984,7 @@ zai.post('/generate-video', async (request, response) => {
19171984router.use('/comfy', comfy);
19181985router.use('/comfyrunpod', comfyRunPod);
19191986router.use('/together', together);
1987+router.use('/sdcpp', sdcpp);
19201988router.use('/drawthings', drawthings);
19211989router.use('/pollinations', pollinations);
19221990router.use('/stability', stability);