Allow returning page if transcript extraction failed

777b2518bdb0153f194db79fb651c558e196fc3d

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 6cb82fc21e3fc7996b5f3c4a1e480890038e47da

src/endpoints/search.js+75 -57 Viewed

     'Sec-Fetch-User': '?1',
 };
 +/**
 + * Extract the transcript of a YouTube video
 + * @param {string} videoPageBody HTML of the video page
 + * @param {string} lang Language code
 + * @returns {Promise<string>} Transcript text
 + */
 +async function extractTranscript(videoPageBody, lang) {
 +    const he = require('he');
 +    const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
 +    const splittedHTML = videoPageBody.split('"captions":');
++
 +    if (splittedHTML.length <= 1) {
 +        if (videoPageBody.includes('class="g-recaptcha"')) {
 +            throw new Error('Too many requests');
 +        }
 +        if (!videoPageBody.includes('"playabilityStatus":')) {
 +            throw new Error('Video is not available');
 +        }
 +        throw new Error('Transcript not available');
 +    }
++
 +    const captions = (() => {
 +        try {
 +            return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
 +        } catch (e) {
 +            return undefined;
 +        }
 +    })()?.['playerCaptionsTracklistRenderer'];
++
 +    if (!captions) {
 +        throw new Error('Transcript disabled');
 +    }
++
 +    if (!('captionTracks' in captions)) {
 +        throw new Error('Transcript not available');
 +    }
++
 +    if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
 +        throw new Error('Transcript not available in this language');
 +    }
++
 +    const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
 +    const transcriptResponse = await fetch(transcriptURL, {
 +        headers: {
 +            ...(lang && { 'Accept-Language': lang }),
 +            'User-Agent': visitHeaders['User-Agent'],
 +        },
 +    });
++
 +    if (!transcriptResponse.ok) {
 +        throw new Error('Transcript request failed');
 +    }
++
 +    const transcriptBody = await transcriptResponse.text();
 +    const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
 +    const transcript = results.map((result) => ({
 +        text: result[3],
 +        duration: parseFloat(result[2]),
 +        offset: parseFloat(result[1]),
 +        lang: lang ?? captions.captionTracks[0].languageCode,
 +    }));
 +    // The text is double-encoded
 +    const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
 +    return transcriptText;
 +}
++
 router.post('/serpapi', jsonParser, async (request, response) => {
     try {
         const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);

  */
 router.post('/transcript', jsonParser, async (request, response) => {
     try {
 -        const he = require('he');
 -        const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
         const id = request.body.id;
         const lang = request.body.lang;
         const json = request.body.json;

         });
         const videoPageBody = await videoPageResponse.text();
 -        const splittedHTML = videoPageBody.split('"captions":');
 -        if (splittedHTML.length <= 1) {
 +        try {
 -            if (videoPageBody.includes('class="g-recaptcha"')) {
 +            const transcriptText = await extractTranscript(videoPageBody, lang);
 -                throw new Error('Too many requests');
 +            return json
 -            }
 +                ? response.json({ transcript: transcriptText, html: videoPageBody })
 -            if (!videoPageBody.includes('"playabilityStatus":')) {
 +                : response.send(transcriptText);
 -                throw new Error('Video is not available');
 +        } catch (error) {
 -            }
 +            if (json) {
 -            throw new Error('Transcript not available');
 +                return response.json({ html: videoPageBody, transcript: '' });
 -        }
+-
 -        const captions = (() => {
 -            try {
 -                return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
 -            } catch (e) {
 -                return undefined;
+            }
 -        })()?.['playerCaptionsTracklistRenderer'];
 +            throw error;
+-
 -        if (!captions) {
 -            throw new Error('Transcript disabled');
 -        }
+-
 -        if (!('captionTracks' in captions)) {
 -            throw new Error('Transcript not available');
 -        }
+-
 -        if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
 -            throw new Error('Transcript not available in this language');
+        }
+-
 -        const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
 -        const transcriptResponse = await fetch(transcriptURL, {
 -            headers: {
 -                ...(lang && { 'Accept-Language': lang }),
 -                'User-Agent': visitHeaders['User-Agent'],
 -            },
 -        });
+-
 -        if (!transcriptResponse.ok) {
 -            throw new Error('Transcript request failed');
 -        }
+-
 -        const transcriptBody = await transcriptResponse.text();
 -        const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
 -        const transcript = results.map((result) => ({
 -            text: result[3],
 -            duration: parseFloat(result[2]),
 -            offset: parseFloat(result[1]),
 -            lang: lang ?? captions.captionTracks[0].languageCode,
 -        }));
 -        // The text is double-encoded
 -        const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
+-
 -        return json
 -            ? response.json({ transcript: transcriptText, html: videoPageBody })
 -            : response.send(transcriptText);
     } catch (error) {
         console.log(error);
         return response.sendStatus(500);