Allow returning page if transcript extraction failed

777b2518bdb0153f194db79fb651c558e196fc3d

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 6cb82fc21e3fc7996b5f3c4a1e480890038e47da
1 files changed, +67 -49Showing whitespace changes
src/endpoints/search.js+67 -49
@@ -22,59 +22,15 @@ const visitHeaders = {
22 'Sec-Fetch-User': '?1',22 'Sec-Fetch-User': '?1',
23};23};
2424
25router.post('/serpapi', jsonParser, async (request, response) => {
26 try {
27 const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
28
29 if (!key) {
30 console.log('No SerpApi key found');
31 return response.sendStatus(400);
32 }
33
34 const { query } = request.body;
35 const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);
36
37 console.log('SerpApi query', query);
38
39 if (!result.ok) {
40 const text = await result.text();
41 console.log('SerpApi request failed', result.statusText, text);
42 return response.status(500).send(text);
43 }
44
45 const data = await result.json();
46 return response.json(data);
47 } catch (error) {
48 console.log(error);
49 return response.sendStatus(500);
50 }
51});
52
53/**25/**
54 * Get the transcript of a YouTube video26 * Extract the transcript of a YouTube video
55 * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)27 * @param {string} videoPageBody HTML of the video page
28 * @param {string} lang Language code
29 * @returns {Promise<string>} Transcript text
56 */30 */
57router.post('/transcript', jsonParser, async (request, response) => {31async function extractTranscript(videoPageBody, lang) {
58 try {
59 const he = require('he');32 const he = require('he');
60 const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;33 const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
61 const id = request.body.id;
62 const lang = request.body.lang;
63 const json = request.body.json;
64
65 if (!id) {
66 console.log('Id is required for /transcript');
67 return response.sendStatus(400);
68 }
69
70 const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
71 headers: {
72 ...(lang && { 'Accept-Language': lang }),
73 'User-Agent': visitHeaders['User-Agent'],
74 },
75 });
76
77 const videoPageBody = await videoPageResponse.text();
78 const splittedHTML = videoPageBody.split('"captions":');34 const splittedHTML = videoPageBody.split('"captions":');
7935
80 if (splittedHTML.length <= 1) {36 if (splittedHTML.length <= 1) {
@@ -129,11 +85,73 @@ router.post('/transcript', jsonParser, async (request, response) => {
129 }));85 }));
130 // The text is double-encoded86 // The text is double-encoded
131 const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');87 const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
88 return transcriptText;
89}
90
91router.post('/serpapi', jsonParser, async (request, response) => {
92 try {
93 const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
94
95 if (!key) {
96 console.log('No SerpApi key found');
97 return response.sendStatus(400);
98 }
99
100 const { query } = request.body;
101 const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);
102
103 console.log('SerpApi query', query);
132104
105 if (!result.ok) {
106 const text = await result.text();
107 console.log('SerpApi request failed', result.statusText, text);
108 return response.status(500).send(text);
109 }
110
111 const data = await result.json();
112 return response.json(data);
113 } catch (error) {
114 console.log(error);
115 return response.sendStatus(500);
116 }
117});
118
119/**
120 * Get the transcript of a YouTube video
121 * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
122 */
123router.post('/transcript', jsonParser, async (request, response) => {
124 try {
125 const id = request.body.id;
126 const lang = request.body.lang;
127 const json = request.body.json;
128
129 if (!id) {
130 console.log('Id is required for /transcript');
131 return response.sendStatus(400);
132 }
133
134 const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
135 headers: {
136 ...(lang && { 'Accept-Language': lang }),
137 'User-Agent': visitHeaders['User-Agent'],
138 },
139 });
140
141 const videoPageBody = await videoPageResponse.text();
142
143 try {
144 const transcriptText = await extractTranscript(videoPageBody, lang);
133 return json145 return json
134 ? response.json({ transcript: transcriptText, html: videoPageBody })146 ? response.json({ transcript: transcriptText, html: videoPageBody })
135 : response.send(transcriptText);147 : response.send(transcriptText);
136 } catch (error) {148 } catch (error) {
149 if (json) {
150 return response.json({ html: videoPageBody, transcript: '' });
151 }
152 throw error;
153 }
154 } catch (error) {
137 console.log(error);155 console.log(error);
138 return response.sendStatus(500);156 return response.sendStatus(500);
139 }157 }