Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · e3f41666 · · 456 lines (14.2 KB)
1 contributor 
1import fetch from 'node-fetch';


2import express from 'express';


3import ipRegex from 'ip-regex';


4


5import { decode } from 'html-entities';


6import { readSecret, SECRET_KEYS } from './secrets.js';


7import { trimV1 } from '../util.js';


8import { setAdditionalHeaders } from '../additional-headers.js';


9


10export const router = express.Router();


11


12// Cosplay as Chrome


13const visitHeaders = {


14    'Accept': 'text/html',


15    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',


16    'Accept-Language': 'en-US,en;q=0.5',


17    'Accept-Encoding': 'gzip, deflate, br',


18    'Connection': 'keep-alive',


19    'Cache-Control': 'no-cache',


20    'Pragma': 'no-cache',


21    'TE': 'trailers',


22    'DNT': '1',


23    'Sec-Fetch-Dest': 'document',


24    'Sec-Fetch-Mode': 'navigate',


25    'Sec-Fetch-Site': 'none',


26    'Sec-Fetch-User': '?1',


27};


28


29/**


30 * Extract the transcript of a YouTube video


31 * @param {string} videoPageBody HTML of the video page


32 * @param {string} lang Language code


33 * @returns {Promise<string>} Transcript text


34 */


35async function extractTranscript(videoPageBody, lang) {


36    const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;


37    const splittedHTML = videoPageBody.split('"captions":');


38


39    if (splittedHTML.length <= 1) {


40        if (videoPageBody.includes('class="g-recaptcha"')) {


41            throw new Error('Too many requests');


42        }


43        if (!videoPageBody.includes('"playabilityStatus":')) {


44            throw new Error('Video is not available');


45        }


46        throw new Error('Transcript not available');


47    }


48


49    const captions = (() => {


50        try {


51            return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));


52        } catch (e) {


53            return undefined;


54        }


55    })()?.playerCaptionsTracklistRenderer;


56


57    if (!captions) {


58        throw new Error('Transcript disabled');


59    }


60


61    if (!('captionTracks' in captions)) {


62        throw new Error('Transcript not available');


63    }


64


65    if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {


66        throw new Error('Transcript not available in this language');


67    }


68


69    const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;


70    const transcriptResponse = await fetch(transcriptURL, {


71        headers: {


72            ...(lang && { 'Accept-Language': lang }),


73            'User-Agent': visitHeaders['User-Agent'],


74        },


75    });


76


77    if (!transcriptResponse.ok) {


78        throw new Error('Transcript request failed');


79    }


80


81    const transcriptBody = await transcriptResponse.text();


82    const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];


83    const transcript = results.map((result) => ({


84        text: result[3],


85        duration: parseFloat(result[2]),


86        offset: parseFloat(result[1]),


87        lang: lang ?? captions.captionTracks[0].languageCode,


88    }));


89    // The text is double-encoded


90    const transcriptText = transcript.map((line) => decode(decode(line.text))).join(' ');


91    return transcriptText;


92}


93


94router.post('/serpapi', async (request, response) => {


95    try {


96        const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);


97


98        if (!key) {


99            console.error('No SerpApi key found');


100            return response.sendStatus(400);


101        }


102


103        const { query } = request.body;


104        const result = await fetch(`https://serpapi.com/search.json?q=${encodeURIComponent(query)}&api_key=${key}`);


105


106        console.debug('SerpApi query', query);


107


108        if (!result.ok) {


109            const text = await result.text();


110            console.error('SerpApi request failed', result.statusText, text);


111            return response.status(500).send(text);


112        }


113


114        const data = await result.json();


115        console.debug('SerpApi response', data);


116        return response.json(data);


117    } catch (error) {


118        console.error(error);


119        return response.sendStatus(500);


120    }


121});


122


123/**


124 * Get the transcript of a YouTube video


125 * @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)


126 */


127router.post('/transcript', async (request, response) => {


128    try {


129        const id = request.body.id;


130        const lang = request.body.lang;


131        const json = request.body.json;


132


133        if (!id) {


134            console.error('Id is required for /transcript');


135            return response.sendStatus(400);


136        }


137


138        const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {


139            headers: {


140                ...(lang && { 'Accept-Language': lang }),


141                'User-Agent': visitHeaders['User-Agent'],


142            },


143        });


144


145        const videoPageBody = await videoPageResponse.text();


146


147        try {


148            const transcriptText = await extractTranscript(videoPageBody, lang);


149            return json


150                ? response.json({ transcript: transcriptText, html: videoPageBody })


151                : response.send(transcriptText);


152        } catch (error) {


153            if (json) {


154                return response.json({ html: videoPageBody, transcript: '' });


155            }


156            throw error;


157        }


158    } catch (error) {


159        console.error(error);


160        return response.sendStatus(500);


161    }


162});


163


164router.post('/searxng', async (request, response) => {


165    try {


166        const { baseUrl, query, preferences, categories } = request.body;


167


168        if (!baseUrl || !query) {


169            console.error('Missing required parameters for /searxng');


170            return response.sendStatus(400);


171        }


172


173        console.debug('SearXNG query', baseUrl, query);


174


175        const mainPageUrl = new URL(baseUrl);


176        const mainPageRequest = await fetch(mainPageUrl, { headers: visitHeaders });


177


178        if (!mainPageRequest.ok) {


179            console.error('SearXNG request failed', mainPageRequest.statusText);


180            return response.sendStatus(500);


181        }


182


183        const mainPageText = await mainPageRequest.text();


184        const clientHref = mainPageText.match(/href="(\/client.+\.css)"/)?.[1];


185


186        if (clientHref) {


187            const clientUrl = new URL(clientHref, baseUrl);


188            await fetch(clientUrl, { headers: visitHeaders });


189        }


190


191        const searchUrl = new URL('/search', baseUrl);


192        const searchParams = new URLSearchParams();


193        searchParams.append('q', query);


194        if (preferences) {


195            searchParams.append('preferences', preferences);


196        }


197        if (categories) {


198            searchParams.append('categories', categories);


199        }


200        searchUrl.search = searchParams.toString();


201


202        const searchResult = await fetch(searchUrl, { headers: visitHeaders });


203


204        if (!searchResult.ok) {


205            const text = await searchResult.text();


206            console.error('SearXNG request failed', searchResult.statusText, text);


207            return response.sendStatus(500);


208        }


209


210        const data = await searchResult.text();


211        return response.send(data);


212    } catch (error) {


213        console.error('SearXNG request failed', error);


214        return response.sendStatus(500);


215    }


216});


217


218router.post('/tavily', async (request, response) => {


219    try {


220        const apiKey = readSecret(request.user.directories, SECRET_KEYS.TAVILY);


221


222        if (!apiKey) {


223            console.error('No Tavily key found');


224            return response.sendStatus(400);


225        }


226


227        const { query, include_images } = request.body;


228


229        const body = {


230            query: query,


231            api_key: apiKey,


232            search_depth: 'basic',


233            topic: 'general',


234            include_answer: true,


235            include_raw_content: false,


236            include_images: !!include_images,


237            include_image_descriptions: false,


238            include_domains: [],


239            max_results: 10,


240        };


241


242        const result = await fetch('https://api.tavily.com/search', {


243            method: 'POST',


244            headers: {


245                'Content-Type': 'application/json',


246            },


247            body: JSON.stringify(body),


248        });


249


250        console.debug('Tavily query', query);


251


252        if (!result.ok) {


253            const text = await result.text();


254            console.error('Tavily request failed', result.statusText, text);


255            return response.status(500).send(text);


256        }


257


258        const data = await result.json();


259        console.debug('Tavily response', data);


260        return response.json(data);


261    } catch (error) {


262        console.error(error);


263        return response.sendStatus(500);


264    }


265});


266


267router.post('/koboldcpp', async (request, response) => {


268    try {


269        const { query, url } = request.body;


270


271        if (!url) {


272            console.error('No URL provided for KoboldCpp search');


273            return response.sendStatus(400);


274        }


275


276        console.debug('KoboldCpp search query', query);


277


278        const baseUrl = trimV1(url);


279        const args = {


280            method: 'POST',


281            headers: {},


282            body: JSON.stringify({ q: query }),


283        };


284


285        setAdditionalHeaders(request, args, baseUrl);


286        const result = await fetch(`${baseUrl}/api/extra/websearch`, args);


287


288        if (!result.ok) {


289            const text = await result.text();


290            console.error('KoboldCpp request failed', result.statusText, text);


291            return response.status(500).send(text);


292        }


293


294        const data = await result.json();


295        console.debug('KoboldCpp search response', data);


296        return response.json(data);


297    } catch (error) {


298        console.error(error);


299        return response.sendStatus(500);


300    }


301});


302


303router.post('/serper', async (request, response) => {


304    try {


305        const key = readSecret(request.user.directories, SECRET_KEYS.SERPER);


306


307        if (!key) {


308            console.error('No Serper key found');


309            return response.sendStatus(400);


310        }


311


312        const { query, images } = request.body;


313


314        const url = images


315            ? 'https://google.serper.dev/images'


316            : 'https://google.serper.dev/search';


317


318        const result = await fetch(url, {


319            method: 'POST',


320            headers: {


321                'X-API-KEY': key,


322                'Content-Type': 'application/json',


323            },


324            redirect: 'follow',


325            body: JSON.stringify({ q: query }),


326        });


327


328        console.debug('Serper query', query);


329


330        if (!result.ok) {


331            const text = await result.text();


332            console.warn('Serper request failed', result.statusText, text);


333            return response.status(500).send(text);


334        }


335


336        const data = await result.json();


337        console.debug('Serper response', data);


338        return response.json(data);


339    } catch (error) {


340        console.error(error);


341        return response.sendStatus(500);


342    }


343});


344


345router.post('/zai', async (request, response) => {


346    try {


347        const key = readSecret(request.user.directories, SECRET_KEYS.ZAI);


348


349        if (!key) {


350            console.error('No Z.AI key found');


351            return response.sendStatus(400);


352        }


353


354        const { query } = request.body;


355


356        if (!query) {


357            console.error('No query provided for /zai');


358            return response.sendStatus(400);


359        }


360


361        console.debug('Z.AI web search query', query);


362


363        const result = await fetch('https://api.z.ai/api/paas/v4/web_search', {


364            method: 'POST',


365            headers: {


366                'Content-Type': 'application/json',


367                'Authorization': `Bearer ${key}`,


368            },


369            body: JSON.stringify({


370                // TODO: There's only one engine option for now


371                search_engine: 'search-prime',


372                search_query: query,


373            }),


374        });


375


376        if (!result.ok) {


377            const text = await result.text();


378            console.error('Z.AI request failed', result.statusText, text);


379            return response.status(500).send(text);


380        }


381


382        const data = await result.json();


383        console.debug('Z.AI web search response', data);


384        return response.json(data);


385    } catch (error) {


386        console.error(error);


387        return response.sendStatus(500);


388    }


389});


390


391router.post('/visit', async (request, response) => {


392    try {


393        const url = request.body.url;


394        const html = Boolean(request.body.html ?? true);


395


396        if (!url) {


397            console.error('No url provided for /visit');


398            return response.sendStatus(400);


399        }


400


401        try {


402            const urlObj = new URL(url);


403


404            // Reject relative URLs


405            if (urlObj.protocol === null || urlObj.host === null) {


406                throw new Error('Invalid URL format');


407            }


408


409            // Reject non-HTTP URLs


410            if (urlObj.protocol !== 'http:' && urlObj.protocol !== 'https:') {


411                throw new Error('Invalid protocol');


412            }


413


414            // Reject URLs with a non-standard port


415            if (urlObj.port !== '') {


416                throw new Error('Invalid port');


417            }


418


419            // Reject IP addresses


420            if (ipRegex.v4({ exact: true }).test(urlObj.hostname) || ipRegex.v6({ exact: true }).test(urlObj.hostname)) {


421                throw new Error('Invalid hostname');


422            }


423        } catch (error) {


424            console.error('Invalid url provided for /visit', url);


425            return response.sendStatus(400);


426        }


427


428        console.info('Visiting web URL', url);


429


430        const result = await fetch(url, { headers: visitHeaders });


431


432        if (!result.ok) {


433            console.error(`Visit failed ${result.status} ${result.statusText}`);


434            return response.sendStatus(500);


435        }


436


437        const contentType = String(result.headers.get('content-type'));


438


439        if (html) {


440            if (!contentType.includes('text/html')) {


441                console.error(`Visit failed, content-type is ${contentType}, expected text/html`);


442                return response.sendStatus(500);


443            }


444


445            const text = await result.text();


446            return response.send(text);


447        }


448


449        response.setHeader('Content-Type', contentType);


450        const buffer = await result.arrayBuffer();


451        return response.send(Buffer.from(buffer));


452    } catch (error) {


453        console.error(error);


454        return response.sendStatus(500);


455    }


456});