Merge pull request #2688 from ayancey/hugging-face-imagegen Hugging Face Inference API Image Generation

296a761247141daa7f79f99a98cbfa804ed34472

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: e3f010782d958f8591fbe2ff85e592cc9a929cc7 a9700c0ff0764ed9fc8e1c93250e92f26df969cd
3 files changed, +110 -1Ignore whitespace
public/scripts/extensions/stable-diffusion/index.js+63 -1
@@ -52,6 +52,7 @@ const sources = {
5252 pollinations: 'pollinations',
5353 stability: 'stability',
5454 blockentropy: 'blockentropy',
55+ huggingface: 'huggingface',
5556};
5657
5758const initiators = {
@@ -454,6 +455,7 @@ async function loadSettings() {
454455 $('#sd_command_visible').prop('checked', extension_settings.sd.command_visible);
455456 $('#sd_interactive_visible').prop('checked', extension_settings.sd.interactive_visible);
456457 $('#sd_stability_style_preset').val(extension_settings.sd.stability_style_preset);
458+ $('#sd_huggingface_model_id').val(extension_settings.sd.huggingface_model_id);
457459
458460 for (const style of extension_settings.sd.styles) {
459461 const option = document.createElement('option');
@@ -1091,6 +1093,11 @@ function onComfyUrlInput() {
10911093 saveSettingsDebounced();
10921094}
10931095
1096+function onHFModelInput() {
1097+ extension_settings.sd.huggingface_model_id = $('#sd_huggingface_model_id').val();
1098+ saveSettingsDebounced();
1099+}
1100+
10941101function onComfyWorkflowChange() {
10951102 extension_settings.sd.comfy_workflow = $('#sd_comfy_workflow').find(':selected').val();
10961103 saveSettingsDebounced();
@@ -1235,7 +1242,16 @@ async function onModelChange() {
12351242 extension_settings.sd.model = $('#sd_model').find(':selected').val();
12361243 saveSettingsDebounced();
12371244
1238- const cloudSources = [sources.horde, sources.novel, sources.openai, sources.togetherai, sources.pollinations, sources.stability, sources.blockentropy];
1245+ const cloudSources = [
1246+ sources.horde,
1247+ sources.novel,
1248+ sources.openai,
1249+ sources.togetherai,
1250+ sources.pollinations,
1251+ sources.stability,
1252+ sources.blockentropy,
1253+ sources.huggingface,
1254+ ];
12391255
12401256 if (cloudSources.includes(extension_settings.sd.source)) {
12411257 return;
@@ -1450,6 +1466,9 @@ async function loadSamplers() {
14501466 case sources.blockentropy:
14511467 samplers = ['N/A'];
14521468 break;
1469+ case sources.huggingface:
1470+ samplers = ['N/A'];
1471+ break;
14531472 }
14541473
14551474 for (const sampler of samplers) {
@@ -1639,6 +1658,9 @@ async function loadModels() {
16391658 case sources.blockentropy:
16401659 models = await loadBlockEntropyModels();
16411660 break;
1661+ case sources.huggingface:
1662+ models = [{ value: '', text: '<Enter Model ID above>' }];
1663+ break;
16421664 }
16431665
16441666 for (const model of models) {
@@ -1986,6 +2008,9 @@ async function loadSchedulers() {
19862008 case sources.blockentropy:
19872009 schedulers = ['N/A'];
19882010 break;
2011+ case sources.huggingface:
2012+ schedulers = ['N/A'];
2013+ break;
19892014 }
19902015
19912016 for (const scheduler of schedulers) {
@@ -2065,6 +2090,9 @@ async function loadVaes() {
20652090 case sources.blockentropy:
20662091 vaes = ['N/A'];
20672092 break;
2093+ case sources.huggingface:
2094+ vaes = ['N/A'];
2095+ break;
20682096 }
20692097
20702098 for (const vae of vaes) {
@@ -2596,6 +2624,9 @@ async function sendGenerationRequest(generationType, prompt, additionalNegativeP
25962624 case sources.blockentropy:
25972625 result = await generateBlockEntropyImage(prefixedPrompt, negativePrompt, signal);
25982626 break;
2627+ case sources.huggingface:
2628+ result = await generateHuggingFaceImage(prefixedPrompt, signal);
2629+ break;
25992630 }
26002631
26012632 if (!result.data) {
@@ -3229,6 +3260,34 @@ async function generateComfyImage(prompt, negativePrompt, signal) {
32293260 return { format: 'png', data: await promptResult.text() };
32303261}
32313262
3263+
3264+/**
3265+ * Generates an image in Hugging Face Inference API using the provided prompt and configuration settings (model selected).
3266+ * @param {string} prompt - The main instruction used to guide the image generation.
3267+ * @param {AbortSignal} signal - An AbortSignal object that can be used to cancel the request.
3268+ * @returns {Promise<{format: string, data: string}>} - A promise that resolves when the image generation and processing are complete.
3269+ */
3270+async function generateHuggingFaceImage(prompt, signal) {
3271+ const result = await fetch('/api/sd/huggingface/generate', {
3272+ method: 'POST',
3273+ headers: getRequestHeaders(),
3274+ signal: signal,
3275+ body: JSON.stringify({
3276+ model: extension_settings.sd.huggingface_model_id,
3277+ prompt: prompt,
3278+ }),
3279+ });
3280+
3281+ if (result.ok) {
3282+ const data = await result.json();
3283+ return { format: 'jpg', data: data.image };
3284+ } else {
3285+ const text = await result.text();
3286+ throw new Error(text);
3287+ }
3288+}
3289+
3290+
32323291async function onComfyOpenWorkflowEditorClick() {
32333292 let workflow = await (await fetch('/api/sd/comfy/workflow', {
32343293 method: 'POST',
@@ -3508,6 +3567,8 @@ function isValidState() {
35083567 return secret_state[SECRET_KEYS.STABILITY];
35093568 case sources.blockentropy:
35103569 return secret_state[SECRET_KEYS.BLOCKENTROPY];
3570+ case sources.huggingface:
3571+ return secret_state[SECRET_KEYS.HUGGINGFACE];
35113572 }
35123573}
35133574
@@ -3848,6 +3909,7 @@ jQuery(async () => {
38483909 $('#sd_swap_dimensions').on('click', onSwapDimensionsClick);
38493910 $('#sd_stability_key').on('click', onStabilityKeyClick);
38503911 $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
3912+ $('#sd_huggingface_model_id').on('input', onHFModelInput);
38513913
38523914 $('.sd_settings .inline-drawer-toggle').on('click', function () {
38533915 initScrollHeight($('#sd_prompt_prefix'));
public/scripts/extensions/stable-diffusion/settings.html+6 -0
@@ -41,6 +41,7 @@
4141 <option value="comfy">ComfyUI</option>
4242 <option value="drawthings">DrawThings HTTP API</option>
4343 <option value="extras">Extras API (local / remote)</option>
44+ <option value="huggingface">HuggingFace Inference API (serverless)</option>
4445 <option value="novel">NovelAI Diffusion</option>
4546 <option value="openai">OpenAI (DALL-E)</option>
4647 <option value="pollinations">Pollinations</option>
@@ -82,6 +83,11 @@
8283 <!-- (Original Text)<b>Important:</b> run DrawThings app with HTTP API switch enabled in the UI! The server must be accessible from the SillyTavern host machine. -->
8384 <i><b data-i18n="Important:">Important:</b></i><i data-i18n="sd_drawthings_auth_txt"> run DrawThings app with HTTP API switch enabled in the UI! The server must be accessible from the SillyTavern host machine.</i>
8485 </div>
86+ <div data-sd-source="huggingface">
87+ <i>Hint: Save an API key in the Hugging Face (Text Completion) API settings to use it here.</i>
88+ <label for="sd_huggingface_model_id" data-i18n="Model ID">Model ID</label>
89+ <input id="sd_huggingface_model_id" type="text" class="text_pole" data-i18n="[placeholder]e.g. black-forest-labs/FLUX.1-dev" placeholder="e.g. black-forest-labs/FLUX.1-dev" value="" />
90+ </div>
8591 <div data-sd-source="vlad">
8692 <label for="sd_vlad_url">SD.Next API URL</label>
8793 <div class="flex-container flexnowrap">
src/endpoints/stable-diffusion.js+41 -0
@@ -991,11 +991,52 @@ blockentropy.post('/generate', jsonParser, async (request, response) => {
991991});
992992
993993
994+const huggingface = express.Router();
995+
996+huggingface.post('/generate', jsonParser, async (request, response) => {
997+ try {
998+ const key = readSecret(request.user.directories, SECRET_KEYS.HUGGINGFACE);
999+
1000+ if (!key) {
1001+ console.log('Hugging Face key not found.');
1002+ return response.sendStatus(400);
1003+ }
1004+
1005+ console.log('Hugging Face request:', request.body);
1006+
1007+ const result = await fetch(`https://api-inference.huggingface.co/models/${request.body.model}`, {
1008+ method: 'POST',
1009+ body: JSON.stringify({
1010+ inputs: request.body.prompt,
1011+ }),
1012+ headers: {
1013+ 'Content-Type': 'application/json',
1014+ 'Authorization': `Bearer ${key}`,
1015+ },
1016+ });
1017+
1018+ if (!result.ok) {
1019+ console.log('Hugging Face returned an error.');
1020+ return response.sendStatus(500);
1021+ }
1022+
1023+ const buffer = await result.buffer();
1024+ return response.send({
1025+ image: buffer.toString('base64'),
1026+ });
1027+ } catch (error) {
1028+ console.log(error);
1029+ return response.sendStatus(500);
1030+ }
1031+});
1032+
1033+
9941034router.use('/comfy', comfy);
9951035router.use('/together', together);
9961036router.use('/drawthings', drawthings);
9971037router.use('/pollinations', pollinations);
9981038router.use('/stability', stability);
9991039router.use('/blockentropy', blockentropy);
1040+router.use('/huggingface', huggingface);
10001041
10011042module.exports = { router };