Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · 51ad27fb · · 432 lines (14.0 KB)
2 contributors 
1import { Buffer } from 'node:buffer';


2import fs from 'node:fs';


3import express from 'express';


4import wavefile from 'wavefile';


5import fetch from 'node-fetch';


6import FormData from 'form-data';


7import mime from 'mime-types';


8import { getPipeline } from '../transformers.js';


9import { forwardFetchResponse } from '../util.js';


10import { readSecret, SECRET_KEYS } from './secrets.js';


11


12export const router = express.Router();


13


14/**


15 * Gets the audio data from a base64-encoded audio file.


16 * @param {string} audio Base64-encoded audio


17 * @returns {Float64Array} Audio data


18 */


19function getWaveFile(audio) {


20    const wav = new wavefile.WaveFile();


21    wav.fromDataURI(audio);


22    wav.toBitDepth('32f');


23    wav.toSampleRate(16000);


24    let audioData = wav.getSamples();


25    if (Array.isArray(audioData)) {


26        if (audioData.length > 1) {


27            const SCALING_FACTOR = Math.sqrt(2);


28


29            // Merge channels (into first channel to save memory)


30            for (let i = 0; i < audioData[0].length; ++i) {


31                audioData[0][i] = SCALING_FACTOR * (audioData[0][i] + audioData[1][i]) / 2;


32            }


33        }


34


35        // Select first channel


36        audioData = audioData[0];


37    }


38


39    return audioData;


40}


41


42router.post('/recognize', async (req, res) => {


43    try {


44        const TASK = 'automatic-speech-recognition';


45        const { model, audio, lang } = req.body;


46        const pipe = await getPipeline(TASK, model);


47        const wav = getWaveFile(audio);


48        const start = performance.now();


49        const result = await pipe(wav, { language: lang || null, task: 'transcribe' });


50        const end = performance.now();


51        console.info(`Execution duration: ${(end - start) / 1000} seconds`);


52        console.info('Transcribed audio:', result.text);


53


54        return res.json({ text: result.text });


55    } catch (error) {


56        console.error(error);


57        return res.sendStatus(500);


58    }


59});


60


61router.post('/synthesize', async (req, res) => {


62    try {


63        const TASK = 'text-to-speech';


64        const { text, model, speaker } = req.body;


65        const pipe = await getPipeline(TASK, model);


66        const speaker_embeddings = speaker


67            ? new Float32Array(new Uint8Array(Buffer.from(speaker.startsWith('data:') ? speaker.split(',')[1] : speaker, 'base64')).buffer)


68            : null;


69        const start = performance.now();


70        const result = await pipe(text, { speaker_embeddings: speaker_embeddings });


71        const end = performance.now();


72        console.debug(`Execution duration: ${(end - start) / 1000} seconds`);


73


74        const wav = new wavefile.WaveFile();


75        wav.fromScratch(1, result.sampling_rate, '32f', result.audio);


76        const buffer = wav.toBuffer();


77


78        res.set('Content-Type', 'audio/wav');


79        return res.send(Buffer.from(buffer));


80    } catch (error) {


81        console.error(error);


82        return res.sendStatus(500);


83    }


84});


85


86const pollinations = express.Router();


87


88pollinations.post('/voices', async (req, res) => {


89    try {


90        const model = req.body.model || 'openai-audio';


91


92        const response = await fetch('https://gen.pollinations.ai/text/models');


93


94        if (!response.ok) {


95            throw new Error('Failed to fetch Pollinations models');


96        }


97


98        const data = await response.json();


99


100        if (!Array.isArray(data)) {


101            throw new Error('Invalid data format received from Pollinations');


102        }


103


104        const audioModelData = data.find(m => m.name === model);


105        if (!audioModelData || !Array.isArray(audioModelData.voices)) {


106            throw new Error('No voices found for the specified model');


107        }


108


109        const voices = audioModelData.voices;


110        return res.json(voices);


111    } catch (error) {


112        console.error(error);


113        return res.sendStatus(500);


114    }


115});


116


117pollinations.post('/generate', async (req, res) => {


118    try {


119        const key = readSecret(req.user.directories, SECRET_KEYS.POLLINATIONS);


120        if (!key) {


121            console.warn('No API key saved for Pollinations TTS.');


122            return res.sendStatus(400);


123        }


124


125        const text = req.body.text;


126        const model = req.body.model || 'openai-audio';


127        const voice = req.body.voice || 'alloy';


128


129        console.debug('Pollinations TTS request', { text, model, voice });


130


131        const response = await fetch('https://gen.pollinations.ai/v1/chat/completions', {


132            method: 'POST',


133            headers: {


134                'Authorization': `Bearer ${key}`,


135                'Content-Type': 'application/json',


136            },


137            body: JSON.stringify({


138                model: model,


139                stream: false,


140                modalities: ['text', 'audio'],


141                seed: Math.floor(Math.random() * Math.pow(2, 32)),


142                audio: {


143                    format: 'mp3',


144                    voice: voice,


145                },


146                messages: [{


147                    role: 'user',


148                    content: text,


149                }],


150            }),


151        });


152


153        if (!response.ok) {


154            const text = await response.text();


155            throw new Error(`Failed to generate audio from Pollinations: ${text}`);


156        }


157


158        /** @type {any} */


159        const data = await response.json();


160        const audioData = data?.choices?.[0]?.message?.audio?.data;


161


162        if (!audioData) {


163            console.warn('Pollinations TTS audio data is missing from the response');


164            return res.sendStatus(500);


165        }


166


167        res.set('Content-Type', 'audio/mpeg');


168        return res.send(Buffer.from(audioData, 'base64'));


169    } catch (error) {


170        console.error(error);


171        return res.sendStatus(500);


172    }


173});


174


175router.use('/pollinations', pollinations);


176


177const elevenlabs = express.Router();


178


179elevenlabs.post('/voices', async (req, res) => {


180    try {


181        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


182        if (!apiKey) {


183            console.warn('ElevenLabs API key not found');


184            return res.sendStatus(400);


185        }


186


187        const response = await fetch('https://api.elevenlabs.io/v1/voices', {


188            headers: {


189                'xi-api-key': apiKey,


190            },


191        });


192


193        if (!response.ok) {


194            const text = await response.text();


195            console.warn(`ElevenLabs voices fetch failed: HTTP ${response.status} - ${text}`);


196            return res.sendStatus(500);


197        }


198


199        const responseJson = await response.json();


200        return res.json(responseJson);


201    } catch (error) {


202        console.error(error);


203        return res.sendStatus(500);


204    }


205});


206


207elevenlabs.post('/voice-settings', async (req, res) => {


208    try {


209        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


210        if (!apiKey) {


211            console.warn('ElevenLabs API key not found');


212            return res.sendStatus(400);


213        }


214


215        const response = await fetch('https://api.elevenlabs.io/v1/voices/settings/default', {


216            headers: {


217                'xi-api-key': apiKey,


218            },


219        });


220


221        if (!response.ok) {


222            const text = await response.text();


223            console.warn(`ElevenLabs voice settings fetch failed: HTTP ${response.status} - ${text}`);


224            return res.sendStatus(500);


225        }


226        const responseJson = await response.json();


227        return res.json(responseJson);


228    } catch (error) {


229        console.error(error);


230        return res.sendStatus(500);


231    }


232});


233


234elevenlabs.post('/synthesize', async (req, res) => {


235    try {


236        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


237        if (!apiKey) {


238            console.warn('ElevenLabs API key not found');


239            return res.sendStatus(400);


240        }


241


242        const { voiceId, request } = req.body;


243


244        if (!voiceId || !request) {


245            console.warn('ElevenLabs synthesis request missing voiceId or request body');


246            return res.sendStatus(400);


247        }


248


249        console.debug('ElevenLabs TTS request:', request);


250


251        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {


252            method: 'POST',


253            headers: {


254                'xi-api-key': apiKey,


255                'Content-Type': 'application/json',


256            },


257            body: JSON.stringify(request),


258        });


259


260        if (!response.ok) {


261            const text = await response.text();


262            console.warn(`ElevenLabs synthesis failed: HTTP ${response.status} - ${text}`);


263            return res.sendStatus(500);


264        }


265


266        res.set('Content-Type', 'audio/mpeg');


267        await forwardFetchResponse(response, res);


268    } catch (error) {


269        console.error(error);


270        return res.sendStatus(500);


271    }


272});


273


274elevenlabs.post('/history', async (req, res) => {


275    try {


276        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


277        if (!apiKey) {


278            console.warn('ElevenLabs API key not found');


279            return res.sendStatus(400);


280        }


281


282        const response = await fetch('https://api.elevenlabs.io/v1/history', {


283            headers: {


284                'xi-api-key': apiKey,


285            },


286        });


287


288        if (!response.ok) {


289            const text = await response.text();


290            console.warn(`ElevenLabs history fetch failed: HTTP ${response.status} - ${text}`);


291            return res.sendStatus(500);


292        }


293


294        const responseJson = await response.json();


295        return res.json(responseJson);


296    } catch (error) {


297        console.error(error);


298        return res.sendStatus(500);


299    }


300});


301


302elevenlabs.post('/history-audio', async (req, res) => {


303    try {


304        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


305        if (!apiKey) {


306            console.warn('ElevenLabs API key not found');


307            return res.sendStatus(400);


308        }


309


310        const { historyItemId } = req.body;


311        if (!historyItemId) {


312            console.warn('ElevenLabs history audio request missing historyItemId');


313            return res.sendStatus(400);


314        }


315


316        console.debug('ElevenLabs history audio request for ID:', historyItemId);


317


318        const response = await fetch(`https://api.elevenlabs.io/v1/history/${historyItemId}/audio`, {


319            headers: {


320                'xi-api-key': apiKey,


321            },


322        });


323


324        if (!response.ok) {


325            const text = await response.text();


326            console.warn(`ElevenLabs history audio fetch failed: HTTP ${response.status} - ${text}`);


327            return res.sendStatus(500);


328        }


329


330        res.set('Content-Type', 'audio/mpeg');


331        await forwardFetchResponse(response, res);


332    } catch (error) {


333        console.error(error);


334        return res.sendStatus(500);


335    }


336});


337


338elevenlabs.post('/voices/add', async (req, res) => {


339    try {


340        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


341        if (!apiKey) {


342            console.warn('ElevenLabs API key not found');


343            return res.sendStatus(400);


344        }


345


346        const { name, description, labels, files } = req.body;


347


348        const formData = new FormData();


349        formData.append('name', name || 'Custom Voice');


350        formData.append('description', description || 'Uploaded via SillyTavern');


351        formData.append('labels', labels || '');


352


353        for (const fileData of (files || [])) {


354            const [mimeType, base64Data] = /^data:(.+);base64,(.+)$/.exec(fileData)?.slice(1) || [];


355            if (!mimeType || !base64Data) {


356                console.warn('Invalid audio file data provided for ElevenLabs voice upload');


357                continue;


358            }


359            const buffer = Buffer.from(base64Data, 'base64');


360            formData.append('files', buffer, {


361                filename: `audio.${mime.extension(mimeType) || 'wav'}`,


362                contentType: mimeType,


363            });


364        }


365


366        console.debug('ElevenLabs voice upload request:', { name, description, labels, files: files?.length || 0 });


367


368        const response = await fetch('https://api.elevenlabs.io/v1/voices/add', {


369            method: 'POST',


370            headers: {


371                'xi-api-key': apiKey,


372            },


373            body: formData,


374        });


375


376        if (!response.ok) {


377            const text = await response.text();


378            console.warn(`ElevenLabs voice upload failed: HTTP ${response.status} - ${text}`);


379            return res.sendStatus(500);


380        }


381


382        const responseJson = await response.json();


383        return res.json(responseJson);


384    } catch (error) {


385        console.error(error);


386        return res.sendStatus(500);


387    }


388});


389


390elevenlabs.post('/recognize', async (req, res) => {


391    try {


392        const apiKey = readSecret(req.user.directories, SECRET_KEYS.ELEVENLABS);


393        if (!apiKey) {


394            console.warn('ElevenLabs API key not found');


395            return res.sendStatus(400);


396        }


397


398        if (!req.file) {


399            console.warn('No audio file found');


400            return res.sendStatus(400);


401        }


402


403        console.info('Processing audio file with ElevenLabs', req.file.path);


404        const formData = new FormData();


405        formData.append('file', fs.createReadStream(req.file.path), { filename: 'audio.wav', contentType: 'audio/wav' });


406        formData.append('model_id', req.body.model);


407


408        const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', {


409            method: 'POST',


410            headers: {


411                'xi-api-key': apiKey,


412            },


413            body: formData,


414        });


415


416        if (!response.ok) {


417            const text = await response.text();


418            console.warn(`ElevenLabs speech recognition failed: HTTP ${response.status} - ${text}`);


419            return res.sendStatus(500);


420        }


421


422        fs.unlinkSync(req.file.path);


423        const responseJson = await response.json();


424        console.debug('ElevenLabs speech recognition response:', responseJson);


425        return res.json(responseJson);


426    } catch (error) {


427        console.error(error);


428        return res.sendStatus(500);


429    }


430});


431


432router.use('/elevenlabs', elevenlabs);