Allow returning page if transcript extraction failed

777b2518bdb0153f194db79fb651c558e196fc3d

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 6cb82fc21e3fc7996b5f3c4a1e480890038e47da

src/endpoints/search.js+75 -57 Viewed

     'Sec-Fetch-User': '?1',
 };
+/**
+ * Extract the transcript of a YouTube video
+ * @param {string} videoPageBody HTML of the video page
+ * @param {string} lang Language code
+ * @returns {Promise<string>} Transcript text
+ */
+async function extractTranscript(videoPageBody, lang) {
+    const he = require('he');
+    const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
+    const splittedHTML = videoPageBody.split('"captions":');
+    if (splittedHTML.length <= 1) {
+        if (videoPageBody.includes('class="g-recaptcha"')) {
+            throw new Error('Too many requests');
+        }
+        if (!videoPageBody.includes('"playabilityStatus":')) {
+            throw new Error('Video is not available');
+        }
+        throw new Error('Transcript not available');
+    }
+    const captions = (() => {
+        try {
+            return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
+        } catch (e) {
+            return undefined;
+        }
+    })()?.['playerCaptionsTracklistRenderer'];
+    if (!captions) {
+        throw new Error('Transcript disabled');
+    }
+    if (!('captionTracks' in captions)) {
+        throw new Error('Transcript not available');
+    }
+    if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
+        throw new Error('Transcript not available in this language');
+    }
+    const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
+    const transcriptResponse = await fetch(transcriptURL, {
+        headers: {
+            ...(lang && { 'Accept-Language': lang }),
+            'User-Agent': visitHeaders['User-Agent'],
+        },
+    });
+    if (!transcriptResponse.ok) {
+        throw new Error('Transcript request failed');
+    }
+    const transcriptBody = await transcriptResponse.text();
+    const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
+    const transcript = results.map((result) => ({
+        text: result[3],
+        duration: parseFloat(result[2]),
+        offset: parseFloat(result[1]),
+        lang: lang ?? captions.captionTracks[0].languageCode,
+    }));
+    // The text is double-encoded
+    const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
+    return transcriptText;
+}
 router.post('/serpapi', jsonParser, async (request, response) => {
     try {
         const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);

  */
 router.post('/transcript', jsonParser, async (request, response) => {
     try {
-        const he = require('he');
-        const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
         const id = request.body.id;
         const lang = request.body.lang;
         const json = request.body.json;

         });
         const videoPageBody = await videoPageResponse.text();
-        const splittedHTML = videoPageBody.split('"captions":');
-        if (splittedHTML.length <= 1) {
+        try {
-            if (videoPageBody.includes('class="g-recaptcha"')) {
+            const transcriptText = await extractTranscript(videoPageBody, lang);
-                throw new Error('Too many requests');
+            return json
-            }
+                ? response.json({ transcript: transcriptText, html: videoPageBody })
-            if (!videoPageBody.includes('"playabilityStatus":')) {
+                : response.send(transcriptText);
-                throw new Error('Video is not available');
+        } catch (error) {
-            }
+            if (json) {
-            throw new Error('Transcript not available');
+                return response.json({ html: videoPageBody, transcript: '' });
-        }
-        const captions = (() => {
-            try {
-                return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
-            } catch (e) {
-                return undefined;
+            }
-        })()?.['playerCaptionsTracklistRenderer'];
+            throw error;
-        if (!captions) {
-            throw new Error('Transcript disabled');
-        }
-        if (!('captionTracks' in captions)) {
-            throw new Error('Transcript not available');
-        }
-        if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
-            throw new Error('Transcript not available in this language');
+        }
-        const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
-        const transcriptResponse = await fetch(transcriptURL, {
-            headers: {
-                ...(lang && { 'Accept-Language': lang }),
-                'User-Agent': visitHeaders['User-Agent'],
-            },
-        });
-        if (!transcriptResponse.ok) {
-            throw new Error('Transcript request failed');
-        }
-        const transcriptBody = await transcriptResponse.text();
-        const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
-        const transcript = results.map((result) => ({
-            text: result[3],
-            duration: parseFloat(result[2]),
-            offset: parseFloat(result[1]),
-            lang: lang ?? captions.captionTracks[0].languageCode,
-        }));
-        // The text is double-encoded
-        const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
-        return json
-            ? response.json({ transcript: transcriptText, html: videoPageBody })
-            : response.send(transcriptText);
     } catch (error) {
         console.log(error);
         return response.sendStatus(500);