Add support for FAL.AI as image gen provider

6e0ed8552ff01f1ce445d5d62c2b261fababd0f8

Kristan Schlikow <kristian@schlikow.de>

Signed

Parents: 1a52d3c29334d45b23667f0f7acf18b39b3cb946

public/scripts/extensions/stable-diffusion/index.js+65 -0 Viewed

     huggingface: 'huggingface',
     nanogpt: 'nanogpt',
     bfl: 'bfl',
 +    falai: 'falai',
 };
 const initiators = {

     return onApiKeyClick('BFL API Key:', SECRET_KEYS.BFL);
+}
 +async function onFalaiKeyClick() {
 +    return onApiKeyClick('FALAI API Key:', SECRET_KEYS.FALAI);
 +}
++
 function onBflUpsamplingInput() {
     extension_settings.sd.bfl_upsampling = !!$('#sd_bfl_upsampling').prop('checked');
     saveSettingsDebounced();

         case sources.bfl:
             models = await loadBflModels();
             break;
 +        case sources.falai:
 +            models = await loadFalaiModels();
 +            break;
+    }
     for (const model of models) {

     ];
+}
 +async function loadFalaiModels() {
 +    $('#sd_falai_key').toggleClass('success', !!secret_state[SECRET_KEYS.FALAI]);
++
 +    const result = await fetch('/api/sd/falai/models', {
 +        method: 'POST',
 +        headers: getRequestHeaders(),
 +    });
++
 +    if (result.ok) {
 +        return await result.json();
 +    }
++
 +    return [];
 +}
++
 async function loadPollinationsModels() {
     const result = await fetch('/api/sd/pollinations/models', {
         method: 'POST',

         case sources.bfl:
             schedulers = ['N/A'];
             break;
 +        case sources.falai:
 +            schedulers = ['N/A'];
 +            break;
+    }
     for (const scheduler of schedulers) {

             case sources.bfl:
                 result = await generateBflImage(prefixedPrompt, signal);
                 break;
 +            case sources.falai:
 +                result = await generateFalaiImage(prefixedPrompt, negativePrompt, signal);
 +                break;
+        }
         if (!result.data) {

+    }
+}
 +/**
 + * Generates an image using the FAL.AI API.
 + * @param {string} prompt - The main instruction used to guide the image generation.
 + * @param {string} negativePrompt - The negative prompt used to guide the image generation.
 + * @param {AbortSignal} signal - An AbortSignal object that can be used to cancel the request.
 + * @returns {Promise<{format: string, data: string}>} - A promise that resolves when the image generation and processing are complete.
 + */
 +async function generateFalaiImage(prompt, negativePrompt, signal) {
 +    const result = await fetch('/api/sd/falai/generate', {
 +        method: 'POST',
 +        headers: getRequestHeaders(),
 +        signal: signal,
 +        body: JSON.stringify({
 +            prompt: prompt,
 +            negative_prompt: negativePrompt,
 +            model: extension_settings.sd.model,
 +            steps: clamp(extension_settings.sd.steps, 1, 50),
 +            guidance: clamp(extension_settings.sd.scale, 1.5, 5),
 +            width: clamp(extension_settings.sd.width, 256, 1440),
 +            height: clamp(extension_settings.sd.height, 256, 1440),
 +            seed: extension_settings.sd.seed >= 0 ? extension_settings.sd.seed : undefined,
 +        }),
 +    });
++
 +    if (result.ok) {
 +        const data = await result.json();
 +        return { format: 'jpg', data: data.image };
 +    } else {
 +        const text = await result.text();
 +        throw new Error(text);
 +    }
 +}
++
 async function onComfyOpenWorkflowEditorClick() {
     let workflow = await (await fetch('/api/sd/comfy/workflow', {
         method: 'POST',

@@ -3782,6 +3844,8 @@ function isValidState() {
3782	3844	return secret_state[SECRET_KEYS.NANOGPT];
3783	3845	case sources.bfl:
3784	3846	return secret_state[SECRET_KEYS.BFL];
	3847	+ case sources.falai:
	3848	+ return secret_state[SECRET_KEYS.FALAI];
3785	3849	}
3786	3850	}
3787	3851

     $('#sd_function_tool').on('input', onFunctionToolInput);
     $('#sd_bfl_key').on('click', onBflKeyClick);
     $('#sd_bfl_upsampling').on('input', onBflUpsamplingInput);
 +    $('#sd_falai_key').on('click', onFalaiKeyClick);
     if (!CSS.supports('field-sizing', 'content')) {
         $('.sd_settings .inline-drawer-toggle').on('click', function () {

public/scripts/extensions/stable-diffusion/settings.html+15 -0 Viewed

                 <option value="auto">Stable Diffusion Web UI (AUTOMATIC1111)</option>
                 <option value="horde">Stable Horde</option>
                 <option value="togetherai">TogetherAI</option>
 +                <option value="falai">FAL.AI</option>
             </select>
             <div data-sd-source="auto">
                 <label for="sd_auto_url">SD Web UI URL</label>

                 </label>
             </div>
 +            <div data-sd-source="falai">
 +                <div class="flex-container flexnowrap alignItemsBaseline marginBot5">
 +                    <a href="https://fal.ai/dashboard" target="_blank" rel="noopener noreferrer">
 +                        <strong data-i18n="API Key">API Key</strong>
 +                        <i class="fa-solid fa-share-from-square"></i>
 +                    </a>
 +                    <span class="expander"></span>
 +                    <div id="sd_falai_key" class="menu_button menu_button_icon">
 +                        <i class="fa-fw fa-solid fa-key"></i>
 +                        <span data-i18n="Click to set">Click to set</span>
 +                    </div>
 +                </div>
 +            </div>
++
             <div class="flex-container">
                 <div class="flex1">
                     <label for="sd_model" data-i18n="Model">Model</label>

public/scripts/secrets.js+1 -0 Viewed

     GENERIC: 'api_key_generic',
     DEEPSEEK: 'api_key_deepseek',
     SERPER: 'api_key_serper',
 +    FALAI: 'api_key_falai',
 };
 const INPUT_MAP = {

src/endpoints/secrets.js+1 -0 Viewed

     TAVILY: 'api_key_tavily',
     NANOGPT: 'api_key_nanogpt',
     BFL: 'api_key_bfl',
 +    FALAI: 'api_key_falai',
     GENERIC: 'api_key_generic',
     DEEPSEEK: 'api_key_deepseek',
     SERPER: 'api_key_serper',

src/endpoints/stable-diffusion.js+120 -0 Viewed

+    }
 });
 +const falai = express.Router();
++
 +falai.post('/models', jsonParser, async (_request, response) => {
 +    try {
 +        const modelsUrl = new URL('https://fal.ai/api/models?categories=text-to-image');
 +        const result = await fetch(modelsUrl);
++
 +        if (!result.ok) {
 +            console.warn('FAL.AI returned an error.', result.status, result.statusText);
 +            throw new Error('FAL.AI request failed.');
 +        }
++
 +        const data = await result.json();
++
 +        if (!Array.isArray(data)) {
 +            console.warn('FAL.AI returned invalid data.');
 +            throw new Error('FAL.AI request failed.');
 +        }
++
 +        const models = data
 +            .filter(x => !x.title.toLowerCase().includes('inpainting') &&
 +                !x.title.toLowerCase().includes('control') &&
 +                !x.title.toLowerCase().includes('upscale'))
 +            .map(x => ({ value: x.modelUrl.split('fal-ai/')[1], text: x.title }));
 +        return response.send(models);
 +    } catch (error) {
 +        console.error(error);
 +        return response.sendStatus(500);
 +    }
 +});
++
 +falai.post('/generate', jsonParser, async (request, response) => {
 +    try {
 +        const key = readSecret(request.user.directories, SECRET_KEYS.FALAI);
++
 +        if (!key) {
 +            console.warn('FAL.AI key not found.');
 +            return response.sendStatus(400);
 +        }
++
 +        const requestBody = {
 +            prompt: request.body.prompt,
 +            image_size: { 'width': request.body.width, 'height': request.body.height },
 +            num_inference_steps: request.body.steps,
 +            seed: request.body.seed ?? null,
 +            guidance_scale: request.body.guidance,
 +            enable_safety_checker: false,
 +        };
++
 +        console.debug('FAL.AI request:', requestBody);
++
 +        const result = await fetch(`https://queue.fal.run/fal-ai/${request.body.model}`, {
 +            method: 'POST',
 +            body: JSON.stringify(requestBody),
 +            headers: {
 +                'Content-Type': 'application/json',
 +                'Authorization': `Key ${key}`,
 +            },
 +        });
++
 +        if (!result.ok) {
 +            console.warn('FAL.AI returned an error.');
 +            return response.sendStatus(500);
 +        }
++
 +        /** @type {any} */
 +        const taskData = await result.json();
 +        const { status_url } = taskData;
++
 +        const MAX_ATTEMPTS = 100;
 +        for (let i = 0; i < MAX_ATTEMPTS; i++) {
 +            await delay(2500);
++
 +            const statusResult = await fetch(status_url, {
 +                headers: {
 +                    'Authorization': `Key ${key}`,
 +                },
 +            });
++
 +            if (!statusResult.ok) {
 +                const text = await statusResult.text();
 +                console.warn('FAL.AI returned an error.', text);
 +                return response.sendStatus(500);
 +            }
++
 +            /** @type {any} */
 +            const statusData = await statusResult.json();
++
 +            if (statusData?.status === 'IN_QUEUE' || statusData?.status === 'IN_PROGRESS') {
 +                continue;
 +            }
++
 +            if (statusData?.status === 'COMPLETED') {
 +                const resultFetch = await fetch(statusData?.response_url, {
 +                    method: 'GET',
 +                    headers: {
 +                        'Authorization': `Key ${key}`,
 +                    },
 +                });
 +                const resultData = await resultFetch.json();
 +                const imageFetch = await fetch(resultData?.images[0].url, {
 +                    headers: {
 +                        'Authorization': `Key ${key}`,
 +                    },
 +                });
++
 +                const fetchData = await imageFetch.arrayBuffer();
 +                const image = Buffer.from(fetchData).toString('base64');
 +                return response.send({ image: image });
 +            }
++
 +            throw new Error('FAL.AI failed to generate image.', { cause: statusData });
 +        }
 +    } catch (error) {
 +        console.error(error);
 +        return response.sendStatus(500);
 +    }
 +});
++
 router.use('/comfy', comfy);
 router.use('/together', together);
 router.use('/drawthings', drawthings);

 router.use('/huggingface', huggingface);
 router.use('/nanogpt', nanogpt);
 router.use('/bfl', bfl);
 +router.use('/falai', falai);