OpenAI: Add Sora 2 API (#4748) * OpenAI: Add Sora 2 API * Add duration control * Support client generation abort * Reduce poll log amount * Simplify selector * Simplify model-specific control handling

b99ccc3f0ca03e4413310d8cc72fe9dc4cb638e6

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: f8c4795b4a9cbcb6f04de2f3369ed9be64ef572d

public/scripts/extensions/stable-diffusion/index.js+57 -0 Viewed

     // OpenAI settings
     openai_style: 'vivid',
     openai_quality: 'standard',
 +    openai_duration: '8',
     style: 'Default',
     styles: defaultStyles,

     $('#sd_interactive_mode').prop('checked', extension_settings.sd.interactive_mode);
     $('#sd_openai_style').val(extension_settings.sd.openai_style);
     $('#sd_openai_quality').val(extension_settings.sd.openai_quality);
 +    $('#sd_openai_duration').val(extension_settings.sd.openai_duration);
     $('#sd_comfy_url').val(extension_settings.sd.comfy_url);
     $('#sd_comfy_prompt').val(extension_settings.sd.comfy_prompt);
     $('#sd_snap').prop('checked', extension_settings.sd.snap);

     saveSettingsDebounced();
+}
 +async function onOpenAiDurationSelect() {
 +    extension_settings.sd.openai_duration = String($('#sd_openai_duration').find(':selected').val());
 +    saveSettingsDebounced();
 +}
++
 async function onViewAnlasClick() {
     const result = await loadNovelSubscriptionData();

         ensureElectronHubQualitySelect(models);
+    }
 +    switchModelSpecificControls(extension_settings.sd.model);
++
     const cloudSources = [
         sources.horde,
         sources.novel,

         ensureElectronHubQualitySelect(models);
+    }
 +    switchModelSpecificControls(extension_settings.sd.model);
++
     for (const model of models) {
         const option = document.createElement('option');
         option.innerText = model.text;

+}
 /**
 + * Show or hide model-specific controls based on the selected model.
 + * @param {string} modelId Model ID
 + */
 +function switchModelSpecificControls(modelId) {
 +    const modelControls = $('.sd_settings [data-sd-model]');
 +    modelControls.hide();
++
 +    if (!modelId) {
 +        return;
 +    }
++
 +    modelControls.each(function () {
 +        const models = String($(this).attr('data-sd-model') || '').split(',').map(m => m.trim());
 +        $(this).toggle(models.includes(modelId));
 +    });
 +}
++
 +/**
  * Ensure the Electron Hub quality select is populated based on the selected model.
  * @param {any[]} models Array of models
  */

         { value: 'gpt-image-1', text: 'gpt-image-1' },
         { value: 'dall-e-3', text: 'dall-e-3' },
         { value: 'dall-e-2', text: 'dall-e-2' },
 +        { value: 'sora-2', text: 'sora-2' },
 +        { value: 'sora-2-pro', text: 'sora-2-pro' },
     ];
+}

     const isDalle2 = extension_settings.sd.model === 'dall-e-2';
     const isDalle3 = extension_settings.sd.model === 'dall-e-3';
     const isGptImg = extension_settings.sd.model === 'gpt-image-1';
 +    const isSora2 = /sora-2/.test(extension_settings.sd.model);
     if (isDalle2 && prompt.length > dalle2PromptLimit) {
         prompt = prompt.substring(0, dalle2PromptLimit);

         height = 512;
+    }
 +    if (isSora2) {
 +        width = aspectRatio >= 1 ? 1280 : 720;
 +        height = aspectRatio >= 1 ? 720 : 1280;
++
 +        const videoResult = await fetch('/api/openai/generate-video', {
 +            method: 'POST',
 +            headers: getRequestHeaders(),
 +            signal: signal,
 +            body: JSON.stringify({
 +                prompt: prompt,
 +                model: extension_settings.sd.model,
 +                size: `${width}x${height}`,
 +                seconds: extension_settings.sd.openai_duration,
 +            }),
 +        });
++
 +        if (!videoResult.ok) {
 +            throw new Error(await videoResult.text());
 +        }
++
 +        const { format, data } = await videoResult.json();
 +        return { format, data };
 +    }
++
     const result = await fetch('/api/openai/generate-image', {
         method: 'POST',
         headers: getRequestHeaders(),

     $('#sd_interactive_mode').on('input', onInteractiveModeInput);
     $('#sd_openai_style').on('change', onOpenAiStyleSelect);
     $('#sd_openai_quality').on('change', onOpenAiQualitySelect);
 +    $('#sd_openai_duration').on('input', onOpenAiDurationSelect);
     $('#sd_multimodal_captioning').on('input', onMultimodalCaptioningInput);
     $('#sd_snap').on('input', onSnapInput);
     $('#sd_clip_skip').on('input', onClipSkipInput);

public/scripts/extensions/stable-diffusion/settings.html+11 -2 Viewed

                 </div>
             </div>
             <div data-sd-source="openai,aimlapi">
 -                <small data-i18n="These settings only apply to DALL-E 3">These settings only apply to DALL-E 3</small>
 +                <div data-sd-model="dall-e-3" class="flex-container">
 -                <div class="flex-container">
                     <div class="flex1">
                         <label for="sd_openai_style" data-i18n="Image Style">Image Style</label>
                         <select id="sd_openai_style">

                         </select>
                     </div>
                 </div>
 +                <div data-sd-model="sora-2,sora-2-pro" class="flex-container">
 +                    <div class="flex1">
 +                        <label for="sd_openai_duration" data-i18n="Duration">Duration</label>
 +                        <select id="sd_openai_duration">
 +                            <option value="4" data-i18n="Short (4 seconds)">Short (4 seconds)</option>
 +                            <option value="8" data-i18n="Medium (8 seconds)">Medium (8 seconds)</option>
 +                            <option value="12" data-i18n="Long (16 seconds)">Long (12 seconds)</option>
 +                        </select>
 +                    </div>
 +                </div>
             </div>
             <div data-sd-source="comfy">
                 <label for="sd_comfy_url">ComfyUI URL</label>

src/endpoints/openai.js+102 -1 Viewed

 import FormData from 'form-data';
 import express from 'express';
 import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1, delay } from '../util.js';
 import { setAdditionalHeaders } from '../additional-headers.js';
 import { readSecret, SECRET_KEYS } from './secrets.js';
 import { AIMLAPI_HEADERS, OPENROUTER_HEADERS } from '../constants.js';

+    }
 });
 +router.post('/generate-video', async (request, response) => {
 +    try {
 +        const controller = new AbortController();
 +        request.socket.removeAllListeners('close');
 +        request.socket.on('close', function () {
 +            controller.abort();
 +        });
++
 +        const key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
++
 +        if (!key) {
 +            console.warn('No OpenAI key found');
 +            return response.sendStatus(400);
 +        }
++
 +        console.debug('OpenAI video generation request', request.body);
++
 +        const videoJobResponse = await fetch('https://api.openai.com/v1/videos', {
 +            method: 'POST',
 +            headers: {
 +                'Content-Type': 'application/json',
 +                'Authorization': `Bearer ${key}`,
 +            },
 +            body: JSON.stringify({
 +                prompt: request.body.prompt,
 +                model: request.body.model || 'sora-2',
 +                size: request.body.size || '720x1280',
 +                seconds: request.body.seconds || '8',
 +            }),
 +        });
++
 +        if (!videoJobResponse.ok) {
 +            const text = await videoJobResponse.text();
 +            console.warn('OpenAI video generation request failed', videoJobResponse.statusText, text);
 +            return response.status(500).send(text);
 +        }
++
 +        /** @type {any} */
 +        const videoJob = await videoJobResponse.json();
++
 +        if (!videoJob || !videoJob.id) {
 +            console.warn('OpenAI video generation returned no job ID', videoJob);
 +            return response.status(500).send('No video job ID returned');
 +        }
++
 +        // Poll for video generation completion
 +        for (let attempt = 0; attempt < 30; attempt++) {
 +            if (controller.signal.aborted) {
 +                console.info('OpenAI video generation aborted by client');
 +                return response.status(500).send('Video generation aborted by client');
 +            }
++
 +            await delay(5000 + attempt * 1000);
 +            console.debug(`Polling OpenAI video job ${videoJob.id}, attempt ${attempt + 1}`);
++
 +            const pollResponse = await fetch(`https://api.openai.com/v1/videos/${videoJob.id}`, {
 +                method: 'GET',
 +                headers: {
 +                    'Authorization': `Bearer ${key}`,
 +                },
 +            });
++
 +            if (!pollResponse.ok) {
 +                const text = await pollResponse.text();
 +                console.warn('OpenAI video job polling failed', pollResponse.statusText, text);
 +                return response.status(500).send(text);
 +            }
++
 +            /** @type {any} */
 +            const pollResult = await pollResponse.json();
 +            console.debug(`OpenAI video job status: ${pollResult.status}, progress: ${pollResult.progress}`);
++
 +            if (pollResult.status === 'failed') {
 +                console.warn('OpenAI video generation failed', pollResult);
 +                return response.status(500).send('Video generation failed');
 +            }
++
 +            if (pollResult.status === 'completed') {
 +                const contentResponse = await fetch(`https://api.openai.com/v1/videos/${videoJob.id}/content`, {
 +                    method: 'GET',
 +                    headers: {
 +                        'Authorization': `Bearer ${key}`,
 +                    },
 +                });
++
 +                if (!contentResponse.ok) {
 +                    const text = await contentResponse.text();
 +                    console.warn('OpenAI video content fetch failed', contentResponse.statusText, text);
 +                    return response.status(500).send(text);
 +                }
++
 +                const contentBuffer = await contentResponse.arrayBuffer();
 +                return response.send({ format: 'mp4', data: Buffer.from(contentBuffer).toString('base64') });
 +            }
 +        }
 +    } catch (error) {
 +        console.error('OpenAI video generation failed', error);
 +        response.status(500).send('Internal server error');
 +    }
 +});
++
 const custom = express.Router();
 custom.post('/generate-voice', async (request, response) => {