ImageGen: add BFL API for image generation

67869364a531b206ae48a27c32585fdf8e57b968

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 54db4983f4663d77db79ec1246888a5791bdb619
5 files changed, +227 -6Ignore whitespace
public/scripts/extensions/stable-diffusion/index.js+87 -5
@@ -39,6 +39,7 @@ const UPDATE_INTERVAL = 1000;
3939// This is a 1x1 transparent PNG
4040const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
4141const CUSTOM_STOP_EVENT = 'sd_stop_generation';
42+const clamp = (value, min, max) => Math.min(Math.max(value, min), max);
4243
4344const sources = {
4445 extras: 'extras',
@@ -55,6 +56,7 @@ const sources = {
5556 blockentropy: 'blockentropy',
5657 huggingface: 'huggingface',
5758 nanogpt: 'nanogpt',
59+ bfl: 'bfl',
5860};
5961
6062const initiators = {
@@ -296,6 +298,9 @@ const defaultSettings = {
296298
297299 // Stability AI settings
298300 stability_style_preset: 'anime',
301+
302+ // BFL API settings
303+ bfl_upsampling: false,
299304};
300305
301306const writePromptFieldsDebounced = debounce(writePromptFields, debounce_timeout.relaxed);
@@ -463,6 +468,7 @@ async function loadSettings() {
463468 $('#sd_stability_style_preset').val(extension_settings.sd.stability_style_preset);
464469 $('#sd_huggingface_model_id').val(extension_settings.sd.huggingface_model_id);
465470 $('#sd_function_tool').prop('checked', extension_settings.sd.function_tool);
471+ $('#sd_bfl_upsampling').prop('checked', extension_settings.sd.bfl_upsampling);
466472
467473 for (const style of extension_settings.sd.styles) {
468474 const option = document.createElement('option');
@@ -1089,15 +1095,14 @@ function onComfyWorkflowChange() {
10891095 saveSettingsDebounced();
10901096}
10911097
10921098async function onStabilityKeyClickonApiKeyClick(popupText, secretKey) {
1093- const popupText = 'Stability AI API Key:';
10941099 const key = await callGenericPopup(popupText, POPUP_TYPE.INPUT, '', {
10951100 customButtons: [{
10961101 text: 'Remove Key',
10971102 appendAtEnd: true,
10981103 result: POPUP_RESULT.NEGATIVE,
10991104 action: async () => {
11001105 await writeSecret(SECRET_KEYS.STABILITYsecretKey, '');
11011106 toastr.success('API Key removed');
11021107 await loadSettingOptions();
11031108 },
@@ -1108,12 +1113,25 @@ async function onStabilityKeyClick() {
11081113 return;
11091114 }
11101115
11111116 await writeSecret(SECRET_KEYS.STABILITYsecretKey, String(key));
11121117
11131118 toastr.success('API Key saved');
11141119 await loadSettingOptions();
11151120}
11161121
1122+async function onStabilityKeyClick() {
1123+ return onApiKeyClick('Stability AI API Key:', SECRET_KEYS.STABILITY);
1124+}
1125+
1126+async function onBflKeyClick() {
1127+ return onApiKeyClick('BFL API Key:', SECRET_KEYS.BFL);
1128+}
1129+
1130+function onBflUpsamplingInput() {
1131+ extension_settings.sd.bfl_upsampling = !!$('#sd_bfl_upsampling').prop('checked');
1132+ saveSettingsDebounced();
1133+}
1134+
11171135function onStabilityStylePresetChange() {
11181136 extension_settings.sd.stability_style_preset = String($('#sd_stability_style_preset').val());
11191137 saveSettingsDebounced();
@@ -1238,6 +1256,7 @@ async function onModelChange() {
12381256 sources.blockentropy,
12391257 sources.huggingface,
12401258 sources.nanogpt,
1259+ sources.bfl,
12411260 ];
12421261
12431262 if (cloudSources.includes(extension_settings.sd.source)) {
@@ -1459,6 +1478,9 @@ async function loadSamplers() {
14591478 case sources.nanogpt:
14601479 samplers = ['N/A'];
14611480 break;
1481+ case sources.bfl:
1482+ samplers = ['N/A'];
1483+ break;
14621484 }
14631485
14641486 for (const sampler of samplers) {
@@ -1654,6 +1676,9 @@ async function loadModels() {
16541676 case sources.nanogpt:
16551677 models = await loadNanoGPTModels();
16561678 break;
1679+ case sources.bfl:
1680+ models = await loadBflModels();
1681+ break;
16571682 }
16581683
16591684 for (const model of models) {
@@ -1680,6 +1705,17 @@ async function loadStabilityModels() {
16801705 ];
16811706}
16821707
1708+async function loadBflModels() {
1709+ $('#sd_bfl_key').toggleClass('success', !!secret_state[SECRET_KEYS.BFL]);
1710+
1711+ return [
1712+ { value: 'flux-pro-1.1-ultra', text: 'flux-pro-1.1-ultra' },
1713+ { value: 'flux-pro-1.1', text: 'flux-pro-1.1' },
1714+ { value: 'flux-pro', text: 'flux-pro' },
1715+ { value: 'flux-dev', text: 'flux-dev' },
1716+ ];
1717+}
1718+
16831719async function loadPollinationsModels() {
16841720 const result = await fetch('/api/sd/pollinations/models', {
16851721 method: 'POST',
@@ -2027,6 +2063,9 @@ async function loadSchedulers() {
20272063 case sources.nanogpt:
20282064 schedulers = ['N/A'];
20292065 break;
2066+ case sources.bfl:
2067+ schedulers = ['N/A'];
2068+ break;
20302069 }
20312070
20322071 for (const scheduler of schedulers) {
@@ -2112,6 +2151,9 @@ async function loadVaes() {
21122151 case sources.nanogpt:
21132152 vaes = ['N/A'];
21142153 break;
2154+ case sources.bfl:
2155+ vaes = ['N/A'];
2156+ break;
21152157 }
21162158
21172159 for (const vae of vaes) {
@@ -2666,6 +2708,10 @@ async function sendGenerationRequest(generationType, prompt, additionalNegativeP
26662708 break;
26672709 case sources.nanogpt:
26682710 result = await generateNanoGPTImage(prefixedPrompt, negativePrompt, signal);
2711+ break;
2712+ case sources.bfl:
2713+ result = await generateBflImage(prefixedPrompt, signal);
2714+ break;
26692715 }
26702716
26712717 if (!result.data) {
@@ -3370,7 +3416,7 @@ async function generateNanoGPTImage(prompt, negativePrompt, signal) {
33703416 width: parseInt(extension_settings.sd.width),
33713417 height: parseInt(extension_settings.sd.height),
33723418 resolution: `${extension_settings.sd.width}x${extension_settings.sd.height}`,
33733419 showExplicitContent: true,
33743420 nImages: 1,
33753421 }),
33763422 });
@@ -3384,6 +3430,38 @@ async function generateNanoGPTImage(prompt, negativePrompt, signal) {
33843430 }
33853431}
33863432
3433+/**
3434+ * Generates an image using the BFL API.
3435+ * @param {string} prompt - The main instruction used to guide the image generation.
3436+ * @param {AbortSignal} signal - An AbortSignal object that can be used to cancel the request.
3437+ * @returns {Promise<{format: string, data: string}>} - A promise that resolves when the image generation and processing are complete.
3438+ */
3439+async function generateBflImage(prompt, signal) {
3440+ const result = await fetch('/api/sd/bfl/generate', {
3441+ method: 'POST',
3442+ headers: getRequestHeaders(),
3443+ signal: signal,
3444+ body: JSON.stringify({
3445+ prompt: prompt,
3446+ model: extension_settings.sd.model,
3447+ steps: clamp(extension_settings.sd.steps, 1, 50),
3448+ guidance: clamp(extension_settings.sd.scale, 1.5, 5),
3449+ width: clamp(extension_settings.sd.width, 256, 1440),
3450+ height: clamp(extension_settings.sd.height, 256, 1440),
3451+ prompt_upsampling: !!extension_settings.sd.bfl_upsampling,
3452+ seed: extension_settings.sd.seed >= 0 ? extension_settings.sd.seed : undefined,
3453+ }),
3454+ });
3455+
3456+ if (result.ok) {
3457+ const data = await result.json();
3458+ return { format: 'jpg', data: data.image };
3459+ } else {
3460+ const text = await result.text();
3461+ throw new Error(text);
3462+ }
3463+}
3464+
33873465async function onComfyOpenWorkflowEditorClick() {
33883466 let workflow = await (await fetch('/api/sd/comfy/workflow', {
33893467 method: 'POST',
@@ -3668,6 +3746,8 @@ function isValidState() {
36683746 return secret_state[SECRET_KEYS.HUGGINGFACE];
36693747 case sources.nanogpt:
36703748 return secret_state[SECRET_KEYS.NANOGPT];
3749+ case sources.bfl:
3750+ return secret_state[SECRET_KEYS.BFL];
36713751 }
36723752}
36733753
@@ -4338,6 +4418,8 @@ jQuery(async () => {
43384418 $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
43394419 $('#sd_huggingface_model_id').on('input', onHFModelInput);
43404420 $('#sd_function_tool').on('input', onFunctionToolInput);
4421+ $('#sd_bfl_key').on('click', onBflKeyClick);
4422+ $('#sd_bfl_upsampling').on('input', onBflUpsamplingInput);
43414423
43424424 if (!CSS.supports('field-sizing', 'content')) {
43434425 $('.sd_settings .inline-drawer-toggle').on('click', function () {
public/scripts/extensions/stable-diffusion/settings.html+23 -1
@@ -37,6 +37,7 @@
3737 </label>
3838 <label for="sd_source" data-i18n="Source">Source</label>
3939 <select id="sd_source">
40+ <option value="bfl">BFL (Black Forest Labs)</option>
4041 <option value="blockentropy">Block Entropy</option>
4142 <option value="comfy">ComfyUI</option>
4243 <option value="drawthings">DrawThings HTTP API</option>
@@ -234,6 +235,27 @@
234235 </div>
235236 </div>
236237 </div>
238+
239+ <div data-sd-source="bfl">
240+ <div class="flex-container flexnowrap alignItemsBaseline marginBot5">
241+ <a href="https://api.bfl.ml/" target="_blank" rel="noopener noreferrer">
242+ <strong data-i18n="API Key">API Key</strong>
243+ <i class="fa-solid fa-share-from-square"></i>
244+ </a>
245+ <span class="expander"></span>
246+ <div id="sd_bfl_key" class="menu_button menu_button_icon">
247+ <i class="fa-fw fa-solid fa-key"></i>
248+ <span data-i18n="Click to set">Click to set</span>
249+ </div>
250+ </div>
251+ <label class="checkbox_label marginBot5" for="sd_bfl_upsampling" title="Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.">
252+ <input id="sd_bfl_upsampling" type="checkbox" />
253+ <span data-i18n="Prompt Upsampling">
254+ Prompt Upsampling
255+ </span>
256+ </label>
257+ </div>
258+
237259 <div class="flex-container">
238260 <div class="flex1">
239261 <label for="sd_model" data-i18n="Model">Model</label>
@@ -385,7 +407,7 @@
385407 </label>
386408 </div>
387409
388410 <div data-sd-source="novel,togetherai,pollinations,comfy,drawthings,vlad,auto,horde,extras,stability,blockentropy,bfl" class="marginTop5">
389411 <label for="sd_seed">
390412 <span data-i18n="Seed">Seed</span>
391413 <small data-i18n="(-1 for random)">(-1 for random)</small>
public/scripts/secrets.js+1 -0
@@ -37,6 +37,7 @@ export const SECRET_KEYS = {
3737 CUSTOM_OPENAI_TTS: 'api_key_custom_openai_tts',
3838 NANOGPT: 'api_key_nanogpt',
3939 TAVILY: 'api_key_tavily',
40+ BFL: 'api_key_bfl',
4041};
4142
4243const INPUT_MAP = {
src/endpoints/secrets.js+1 -0
@@ -49,6 +49,7 @@ export const SECRET_KEYS = {
4949 CUSTOM_OPENAI_TTS: 'api_key_custom_openai_tts',
5050 TAVILY: 'api_key_tavily',
5151 NANOGPT: 'api_key_nanogpt',
52+ BFL: 'api_key_bfl',
5253};
5354
5455// These are the keys that are safe to expose, even if allowKeysExposure is false
src/endpoints/stable-diffusion.js+115 -0
@@ -1101,6 +1101,120 @@ nanogpt.post('/generate', jsonParser, async (request, response) => {
11011101 }
11021102});
11031103
1104+const bfl = express.Router();
1105+
1106+bfl.post('/generate', jsonParser, async (request, response) => {
1107+ try {
1108+ const key = readSecret(request.user.directories, SECRET_KEYS.BFL);
1109+
1110+ if (!key) {
1111+ console.log('BFL key not found.');
1112+ return response.sendStatus(400);
1113+ }
1114+
1115+ const requestBody = {
1116+ prompt: request.body.prompt,
1117+ steps: request.body.steps,
1118+ guidance: request.body.guidance,
1119+ width: request.body.width,
1120+ height: request.body.height,
1121+ prompt_upsampling: request.body.prompt_upsampling,
1122+ seed: request.body.seed ?? null,
1123+ safety_tolerance: 6, // being least strict
1124+ output_format: 'jpeg',
1125+ };
1126+
1127+ function getClosestAspectRatio(width, height) {
1128+ const minAspect = 9 / 21;
1129+ const maxAspect = 21 / 9;
1130+ const currentAspect = width / height;
1131+
1132+ const gcd = (a, b) => b === 0 ? a : gcd(b, a % b);
1133+ const simplifyRatio = (w, h) => {
1134+ const divisor = gcd(w, h);
1135+ return `${w / divisor}:${h / divisor}`;
1136+ };
1137+
1138+ if (currentAspect < minAspect) {
1139+ const adjustedHeight = Math.round(width / minAspect);
1140+ return simplifyRatio(width, adjustedHeight);
1141+ } else if (currentAspect > maxAspect) {
1142+ const adjustedWidth = Math.round(height * maxAspect);
1143+ return simplifyRatio(adjustedWidth, height);
1144+ } else {
1145+ return simplifyRatio(width, height);
1146+ }
1147+ }
1148+
1149+ if (String(request.body.model).endsWith('-ultra')) {
1150+ requestBody.aspect_ratio = getClosestAspectRatio(request.body.width, request.body.height);
1151+ delete requestBody.steps;
1152+ delete requestBody.guidance;
1153+ delete requestBody.width;
1154+ delete requestBody.height;
1155+ delete requestBody.prompt_upsampling;
1156+ }
1157+
1158+ if (String(request.body.model).endsWith('-pro-1.1')) {
1159+ delete requestBody.steps;
1160+ delete requestBody.guidance;
1161+ }
1162+
1163+ console.log('BFL request:', requestBody);
1164+
1165+ const result = await fetch(`https://api.bfl.ml/v1/${request.body.model}`, {
1166+ method: 'POST',
1167+ body: JSON.stringify(requestBody),
1168+ headers: {
1169+ 'Content-Type': 'application/json',
1170+ 'x-key': key,
1171+ },
1172+ });
1173+
1174+ if (!result.ok) {
1175+ console.log('BFL returned an error.');
1176+ return response.sendStatus(500);
1177+ }
1178+
1179+ /** @type {any} */
1180+ const taskData = await result.json();
1181+ const { id } = taskData;
1182+
1183+ const MAX_ATTEMPTS = 100;
1184+ for (let i = 0; i < MAX_ATTEMPTS; i++) {
1185+ await delay(2500);
1186+
1187+ const statusResult = await fetch(`https://api.bfl.ml/v1/get_result?id=${id}`);
1188+
1189+ if (!statusResult.ok) {
1190+ const text = await statusResult.text();
1191+ console.log('BFL returned an error.', text);
1192+ return response.sendStatus(500);
1193+ }
1194+
1195+ /** @type {any} */
1196+ const statusData = await statusResult.json();
1197+
1198+ if (statusData?.status === 'Pending') {
1199+ continue;
1200+ }
1201+
1202+ if (statusData?.status === 'Ready') {
1203+ const { sample } = statusData.result;
1204+ const fetchResult = await fetch(sample);
1205+ const fetchData = await fetchResult.arrayBuffer();
1206+ const image = Buffer.from(fetchData).toString('base64');
1207+ return response.send({ image: image });
1208+ }
1209+
1210+ throw new Error('BFL failed to generate image.', { cause: statusData });
1211+ }
1212+ } catch (error) {
1213+ console.log(error);
1214+ return response.sendStatus(500);
1215+ }
1216+});
1217+
11041218router.use('/comfy', comfy);
11051219router.use('/together', together);
11061220router.use('/drawthings', drawthings);
@@ -1109,3 +1223,4 @@ router.use('/stability', stability);
11091223router.use('/blockentropy', blockentropy);
11101224router.use('/huggingface', huggingface);
11111225router.use('/nanogpt', nanogpt);
1226+router.use('/bfl', bfl);