OpenAI: Add Sora 2 API (#4748) * OpenAI: Add Sora 2 API * Add duration control * Support client generation abort * Reduce poll log amount * Simplify selector * Simplify model-specific control handling

b99ccc3f0ca03e4413310d8cc72fe9dc4cb638e6

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: f8c4795b4a9cbcb6f04de2f3369ed9be64ef572d

public/scripts/extensions/stable-diffusion/index.js+57 -0 Viewed

     // OpenAI settings
     openai_style: 'vivid',
     openai_quality: 'standard',
+    openai_duration: '8',
     style: 'Default',
     styles: defaultStyles,

     $('#sd_interactive_mode').prop('checked', extension_settings.sd.interactive_mode);
     $('#sd_openai_style').val(extension_settings.sd.openai_style);
     $('#sd_openai_quality').val(extension_settings.sd.openai_quality);
+    $('#sd_openai_duration').val(extension_settings.sd.openai_duration);
     $('#sd_comfy_url').val(extension_settings.sd.comfy_url);
     $('#sd_comfy_prompt').val(extension_settings.sd.comfy_prompt);
     $('#sd_snap').prop('checked', extension_settings.sd.snap);

     saveSettingsDebounced();
+}
+async function onOpenAiDurationSelect() {
+    extension_settings.sd.openai_duration = String($('#sd_openai_duration').find(':selected').val());
+    saveSettingsDebounced();
+}
 async function onViewAnlasClick() {
     const result = await loadNovelSubscriptionData();

         ensureElectronHubQualitySelect(models);
+    }
+    switchModelSpecificControls(extension_settings.sd.model);
     const cloudSources = [
         sources.horde,
         sources.novel,

         ensureElectronHubQualitySelect(models);
+    }
+    switchModelSpecificControls(extension_settings.sd.model);
     for (const model of models) {
         const option = document.createElement('option');
         option.innerText = model.text;

+}
 /**
+ * Show or hide model-specific controls based on the selected model.
+ * @param {string} modelId Model ID
+ */
+function switchModelSpecificControls(modelId) {
+    const modelControls = $('.sd_settings [data-sd-model]');
+    modelControls.hide();
+    if (!modelId) {
+        return;
+    }
+    modelControls.each(function () {
+        const models = String($(this).attr('data-sd-model') || '').split(',').map(m => m.trim());
+        $(this).toggle(models.includes(modelId));
+    });
+}
+/**
  * Ensure the Electron Hub quality select is populated based on the selected model.
  * @param {any[]} models Array of models
  */

         { value: 'gpt-image-1', text: 'gpt-image-1' },
         { value: 'dall-e-3', text: 'dall-e-3' },
         { value: 'dall-e-2', text: 'dall-e-2' },
+        { value: 'sora-2', text: 'sora-2' },
+        { value: 'sora-2-pro', text: 'sora-2-pro' },
     ];
+}

     const isDalle2 = extension_settings.sd.model === 'dall-e-2';
     const isDalle3 = extension_settings.sd.model === 'dall-e-3';
     const isGptImg = extension_settings.sd.model === 'gpt-image-1';
+    const isSora2 = /sora-2/.test(extension_settings.sd.model);
     if (isDalle2 && prompt.length > dalle2PromptLimit) {
         prompt = prompt.substring(0, dalle2PromptLimit);

         height = 512;
+    }
+    if (isSora2) {
+        width = aspectRatio >= 1 ? 1280 : 720;
+        height = aspectRatio >= 1 ? 720 : 1280;
+        const videoResult = await fetch('/api/openai/generate-video', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            signal: signal,
+            body: JSON.stringify({
+                prompt: prompt,
+                model: extension_settings.sd.model,
+                size: `${width}x${height}`,
+                seconds: extension_settings.sd.openai_duration,
+            }),
+        });
+        if (!videoResult.ok) {
+            throw new Error(await videoResult.text());
+        }
+        const { format, data } = await videoResult.json();
+        return { format, data };
+    }
     const result = await fetch('/api/openai/generate-image', {
         method: 'POST',
         headers: getRequestHeaders(),

     $('#sd_interactive_mode').on('input', onInteractiveModeInput);
     $('#sd_openai_style').on('change', onOpenAiStyleSelect);
     $('#sd_openai_quality').on('change', onOpenAiQualitySelect);
+    $('#sd_openai_duration').on('input', onOpenAiDurationSelect);
     $('#sd_multimodal_captioning').on('input', onMultimodalCaptioningInput);
     $('#sd_snap').on('input', onSnapInput);
     $('#sd_clip_skip').on('input', onClipSkipInput);

public/scripts/extensions/stable-diffusion/settings.html+11 -2 Viewed

                 </div>
             </div>
             <div data-sd-source="openai,aimlapi">
-                <small data-i18n="These settings only apply to DALL-E 3">These settings only apply to DALL-E 3</small>
+                <div data-sd-model="dall-e-3" class="flex-container">
-                <div class="flex-container">
                     <div class="flex1">
                         <label for="sd_openai_style" data-i18n="Image Style">Image Style</label>
                         <select id="sd_openai_style">

                         </select>
                     </div>
                 </div>
+                <div data-sd-model="sora-2,sora-2-pro" class="flex-container">
+                    <div class="flex1">
+                        <label for="sd_openai_duration" data-i18n="Duration">Duration</label>
+                        <select id="sd_openai_duration">
+                            <option value="4" data-i18n="Short (4 seconds)">Short (4 seconds)</option>
+                            <option value="8" data-i18n="Medium (8 seconds)">Medium (8 seconds)</option>
+                            <option value="12" data-i18n="Long (16 seconds)">Long (12 seconds)</option>
+                        </select>
+                    </div>
+                </div>
             </div>
             <div data-sd-source="comfy">
                 <label for="sd_comfy_url">ComfyUI URL</label>

src/endpoints/openai.js+102 -1 Viewed

 import FormData from 'form-data';
 import express from 'express';
-import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1 } from '../util.js';
+import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1, delay } from '../util.js';
 import { setAdditionalHeaders } from '../additional-headers.js';
 import { readSecret, SECRET_KEYS } from './secrets.js';
 import { AIMLAPI_HEADERS, OPENROUTER_HEADERS } from '../constants.js';

+    }
 });
+router.post('/generate-video', async (request, response) => {
+    try {
+        const controller = new AbortController();
+        request.socket.removeAllListeners('close');
+        request.socket.on('close', function () {
+            controller.abort();
+        });
+        const key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
+        if (!key) {
+            console.warn('No OpenAI key found');
+            return response.sendStatus(400);
+        }
+        console.debug('OpenAI video generation request', request.body);
+        const videoJobResponse = await fetch('https://api.openai.com/v1/videos', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${key}`,
+            },
+            body: JSON.stringify({
+                prompt: request.body.prompt,
+                model: request.body.model || 'sora-2',
+                size: request.body.size || '720x1280',
+                seconds: request.body.seconds || '8',
+            }),
+        });
+        if (!videoJobResponse.ok) {
+            const text = await videoJobResponse.text();
+            console.warn('OpenAI video generation request failed', videoJobResponse.statusText, text);
+            return response.status(500).send(text);
+        }
+        /** @type {any} */
+        const videoJob = await videoJobResponse.json();
+        if (!videoJob || !videoJob.id) {
+            console.warn('OpenAI video generation returned no job ID', videoJob);
+            return response.status(500).send('No video job ID returned');
+        }
+        // Poll for video generation completion
+        for (let attempt = 0; attempt < 30; attempt++) {
+            if (controller.signal.aborted) {
+                console.info('OpenAI video generation aborted by client');
+                return response.status(500).send('Video generation aborted by client');
+            }
+            await delay(5000 + attempt * 1000);
+            console.debug(`Polling OpenAI video job ${videoJob.id}, attempt ${attempt + 1}`);
+            const pollResponse = await fetch(`https://api.openai.com/v1/videos/${videoJob.id}`, {
+                method: 'GET',
+                headers: {
+                    'Authorization': `Bearer ${key}`,
+                },
+            });
+            if (!pollResponse.ok) {
+                const text = await pollResponse.text();
+                console.warn('OpenAI video job polling failed', pollResponse.statusText, text);
+                return response.status(500).send(text);
+            }
+            /** @type {any} */
+            const pollResult = await pollResponse.json();
+            console.debug(`OpenAI video job status: ${pollResult.status}, progress: ${pollResult.progress}`);
+            if (pollResult.status === 'failed') {
+                console.warn('OpenAI video generation failed', pollResult);
+                return response.status(500).send('Video generation failed');
+            }
+            if (pollResult.status === 'completed') {
+                const contentResponse = await fetch(`https://api.openai.com/v1/videos/${videoJob.id}/content`, {
+                    method: 'GET',
+                    headers: {
+                        'Authorization': `Bearer ${key}`,
+                    },
+                });
+                if (!contentResponse.ok) {
+                    const text = await contentResponse.text();
+                    console.warn('OpenAI video content fetch failed', contentResponse.statusText, text);
+                    return response.status(500).send(text);
+                }
+                const contentBuffer = await contentResponse.arrayBuffer();
+                return response.send({ format: 'mp4', data: Buffer.from(contentBuffer).toString('base64') });
+            }
+        }
+    } catch (error) {
+        console.error('OpenAI video generation failed', error);
+        response.status(500).send('Internal server error');
+    }
+});
 const custom = express.Router();
 custom.post('/generate-voice', async (request, response) => {