Implement private IP range request host validator (#5497) * feat: implement private IP range request host validator for server-side HTTP requests * feat: add link-local address support * fix: use correct config keys * fix: if config missing use default loopback addresses * fix: re-use resolved address for connection * test: add unit coverage for private request filter and proxy interaction Agent-Logs-Url: https://github.com/SillyTavern/SillyTavern/sessions/1813593e-2263-45e2-aa53-74d39515f1df Co-authored-by: Cohee1207 <18619528+Cohee1207@users.noreply.github.com> * test: remove request-proxy.test.js * perf: cache resolved matches * fix: remove unused import * fix: use proper ipv4 loopback cidr * fix: correct raiseError comment * test: uses tls.connect for secure endpoints * Implement private IP range request host validator Agent-Logs-Url: https://github.com/SillyTavern/SillyTavern/sessions/e76ba122-136e-43ad-b4bc-ea48a01fcdda Co-authored-by: Cohee1207 <18619528+Cohee1207@users.noreply.github.com> * Revert "Implement private IP range request host validator" This reverts commit 14e271470227b485b7d23caac31a237abf9f7835. * fix: close request without sending status in CORS forwarding when headers were sent * fix: not enabled -> disabled * feat: add enableKeepAlive option to PrivateRequestAgent Co-authored-by: Copilot <copilot@github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: anthropic-code-agent[bot] <242468646+Claude@users.noreply.github.com> Co-authored-by: Copilot <copilot@github.com>

338119ab77309d5665d9d649377b81890829e158

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

Parents: 1bb2a5ea19f95a2b1918dbf979e894ed3eb23451

default/config.yaml+20 -0 Viewed

   # - .trycloudflare.com
   hosts: []
 +# Perform whitelist checks against server-side HTTP requests that resolve to private IP addresses.
 +# This is an additional layer of security to prevent Server-Side Request Forgery (SSRF) attacks.
 +# Recommended when listen mode is enabled, or if your server is accessible by untrusted users.
 +privateAddressWhitelist:
 +  # Enable private address whitelist to block requests to private IP ranges.
 +  enabled: false
 +  # If true, requests to hosts that cannot be resolved will be allowed instead of blocked.
 +  allowUnresolvedHosts: false
 +  # Log blocked and allowed requests to the console.
 +  log:
 +    # Log blocked requests to the console with a warning message
 +    blockedRequests: true
 +    # Log allowed requests to the console with an info message
 +    allowedRequests: false
 +  # List of allowed private IP ranges (in CIDR notation or wildcard format).
 +  # Allows loopback IP ranges by default, but you can customize this list to fit your needs.
 +  allowedRanges:
 +    - '127.0.0.0/8'      # Loopback (IPv4)
 +    - '::1/128'          # Loopback (IPv6)
++
 # User session timeout *in seconds* (defaults to 24 hours).
 ## Set to a positive number to expire session after a certain time of inactivity
 ## Set to 0 to expire session when the browser is closed

package-lock.json+1 -0 Viewed

                 "@mozilla/readability": "^0.6.0",
                 "@popperjs/core": "^2.11.8",
                 "@zeldafan0225/ai_horde": "^5.2.0",
 +                "agent-base": "^7.1.3",
                 "archiver": "^7.0.1",
                 "bing-translate-api": "^4.1.0",
                 "body-parser": "^1.20.2",

package.json+1 -0 Viewed

         "@mozilla/readability": "^0.6.0",
         "@popperjs/core": "^2.11.8",
         "@zeldafan0225/ai_horde": "^5.2.0",
 +        "agent-base": "^7.1.3",
         "archiver": "^7.0.1",
         "bing-translate-api": "^4.1.0",
         "body-parser": "^1.20.2",

src/middleware/corsProxy.js+5 -1 Viewed

         // Copy over relevant response params to the proxy response
         await forwardFetchResponse(response, res);
     } catch (error) {
 -        res.status(500).send('Error occurred while trying to proxy to: ' + url + ' ' + error);
 +        console.error('Error in CORS proxy middleware:', error);
 +        if (!res.headersSent) {
 +            return res.sendStatus(500);
 +        }
 +        return res.end();
+    }
+}

src/private-request-filter.js+231 -0 Viewed

 +import net from 'node:net';
 +import tls from 'node:tls';
 +import http from 'node:http';
 +import https from 'node:https';
 +import dns from 'node:dns';
 +import ipMatch from 'ip-matching';
 +import ipRegex from 'ip-regex';
 +import { Agent } from 'agent-base';
 +import { color } from './util.js';
 +import { filterValidIpPatterns } from './express-common.js';
++
 +const LOG_HEADER = '[Private Request Filter]';
++
 +/** @type {import('ip-matching').IPMatch[]} */
 +const privateIpRanges = [
 +    // Loopback (IPv4)
 +    ipMatch.getMatch('127.0.0.0/8'),
 +    // Class A private network
 +    ipMatch.getMatch('10.0.0.0/8'),
 +    // Class B private network
 +    ipMatch.getMatch('172.16.0.0/12'),
 +    // Class C private network
 +    ipMatch.getMatch('192.168.0.0/16'),
 +    // Link-local address (IPv4)
 +    ipMatch.getMatch('169.254.0.0/16'),
 +    // Loopback (IPv6)
 +    ipMatch.getMatch('::1/128'),
 +    // Unique local address (IPv6)
 +    ipMatch.getMatch('fc00::/7'),
 +    // Link-local address (IPv6)
 +    ipMatch.getMatch('fe80::/10'),
 +];
++
 +/**
 + * Custom HTTP/HTTPS agent that blocks requests to private IP addresses unless they are explicitly allowed in the private address whitelist.
 + * This is used to prevent Server-Side Request Forgery (SSRF) attacks by ensuring that the server cannot make requests to internal services or resources that are not intended to be exposed.
 + * The agent checks if the target host resolves to a private IP address and blocks the request if it does, unless the IP address is included in the private address whitelist.
 + * The private address whitelist can contain specific IP addresses or CIDR ranges that are allowed to be accessed even if they fall within private IP ranges.
 + */
 +class PrivateRequestAgent extends Agent {
 +    /**
 +     * List of private IP addresses or CIDR ranges to allow
 +     * @type {Readonly<import('ip-matching').IPMatch[]>}
 +     */
 +    privateAddressWhitelist = [];
++
 +    /**
 +     * Whether to log blocked requests to the console
 +     * @type {boolean}
 +     */
 +    logBlocked = true;
++
 +    /**
 +     * Whether to log allowed requests to the console
 +     * @type {boolean}
 +     */
 +    logAllowed = false;
++
 +    /**
 +     * Whether to allow requests to hosts that cannot be resolved
 +     * @type {boolean}
 +     */
 +    allowUnresolvedHosts = false;
++
 +    /**
 +     * Create a new PrivateRequestAgent instance.
 +     * @param {object} options
 +     * @param {string[]} options.privateAddressWhitelist List of private IP addresses or CIDR ranges to allow.
 +     * @param {boolean} options.logBlocked Whether to log blocked requests to the console.
 +     * @param {boolean} options.logAllowed Whether to log allowed requests to the console.
 +     * @param {boolean} options.allowUnresolvedHosts Whether to allow requests to hosts that cannot be resolved.
 +     * @param {boolean} options.enableKeepAlive Whether to enable HTTP/HTTPS keep-alive.
 +     */
 +    constructor(options = { privateAddressWhitelist: [], logBlocked: true, logAllowed: false, allowUnresolvedHosts: false, enableKeepAlive: false }) {
 +        super({ keepAlive: options.enableKeepAlive });
++
 +        const logEntryWarning = (entry, message) => `${color.red('Warning')}: Ignoring invalid private whitelist entry ${color.yellow(entry)} - ${message}`;
 +        const whitelistArray = Array.isArray(options.privateAddressWhitelist) ? options.privateAddressWhitelist : [];
 +        this.privateAddressWhitelist = Object.freeze(filterValidIpPatterns(whitelistArray, logEntryWarning).map(pattern => ipMatch.getMatch(pattern)));
 +        this.allowUnresolvedHosts = options.allowUnresolvedHosts;
 +        this.logBlocked = options.logBlocked;
 +        this.logAllowed = options.logAllowed;
 +    }
++
 +    /**
 +     * Check if the given address is a private IP address.
 +     * @param {string} address The IP address to check.
 +     * @returns {boolean} Whether the given address is a private IP address.
 +     */
 +    #isPrivateIp(address) {
 +        return privateIpRanges.some(range => range.matches(address));
 +    }
++
 +    /**
 +     * Check if the given address is allowed based on the private address whitelist.
 +     * @param {string} address The IP address to check.
 +     * @returns {boolean} Whether the given address is allowed based on the private address whitelist.
 +     */
 +    #isAllowedPrivateAddress(address) {
 +        // Permit the request if the private IP address is in the whitelist
 +        return this.privateAddressWhitelist.some(match => match.matches(address));
 +    }
++
 +    /**
 +     * Connect method that checks if the target host resolves to a private IP address and blocks the request if it does.
 +     * @param {http.ClientRequest} _req HTTP request object.
 +     * @param {import('agent-base').AgentConnectOpts} options Agent connection options.
 +     */
 +    async connect(_req, options) {
 +        /**
 +         * Raise an error and log it if necessary.
 +         * @param {string} message The error message.
 +         * @param {boolean} [log=true] Whether to log the error to the console.
 +         */
 +        const raiseError = (message, log = true) => {
 +            if (log) {
 +                console.error(color.red(LOG_HEADER), message);
 +            }
 +            throw new Error(message);
 +        };
++
 +        /**
 +         * Establish a connection to the target host using either TLS or a regular socket based on the options provided.
 +         * @param {string|null} [hostOverride] Pass a host to override the one in options when connecting.
 +         * @returns {net.Socket|tls.TLSSocket} A socket connected to the target host.
 +         */
 +        const connect = (hostOverride = null) => {
 +            if (hostOverride) {
 +                options.host = hostOverride;
 +            }
 +            if (options.secureEndpoint) {
 +                return tls.connect(options);
 +            } else {
 +                return net.connect(options);
 +            }
 +        };
++
 +        /**
 +         * Validate the given IP address against the private address whitelist and connect if it's allowed.
 +         * @param {string} ip The IP address to validate.
 +         * @returns {net.Socket|tls.TLSSocket} A socket connected to the target IP address if it's allowed, otherwise an error is raised.
 +         */
 +        const validateIpAddress = (ip) => {
 +            // Not a private IP address, allow the request
 +            if (!this.#isPrivateIp(ip)) {
 +                return connect(ip);
 +            }
++
 +            // Private IP address, check if it's allowed in the whitelist
 +            if (this.#isAllowedPrivateAddress(ip)) {
 +                if (this.logAllowed) {
 +                    console.info(color.green(LOG_HEADER), 'Allowed request to private IP address:', color.blue(ip));
 +                }
++
 +                return connect(ip);
 +            }
++
 +            return raiseError(`Blocked request to private IP address: ${ip}`, this.logBlocked);
 +        };
++
 +        /**
 +         * Resolve the given host to an IP address using DNS lookup.
 +         * @param {string} host The host to resolve to an IP address.
 +         * @returns {Promise<string>} The resolved IP address for the given host, or an empty string if the host cannot be resolved.
 +         */
 +        const lookupHost = async (host) => {
 +            try {
 +                return (await dns.promises.lookup(host)).address;
 +            } catch {
 +                return '';
 +            }
 +        };
++
 +        const host = options.host;
++
 +        if (!host) {
 +            return raiseError('No host specified in request options', true);
 +        }
++
 +        const isIp = ipRegex.v4({ exact: true }).test(host) || ipRegex.v6({ exact: true }).test(host);
++
 +        if (isIp) {
 +            return validateIpAddress(host);
 +        } else {
 +            const address = await lookupHost(host);
 +            if (!address) {
 +                if (this.allowUnresolvedHosts) {
 +                    return connect();
 +                } else {
 +                    return raiseError(`Unable to resolve host: ${host}. Set privateAddressWhitelist.allowUnresolvedHosts to true to bypass this check.`, true);
 +                }
 +            }
++
 +            return validateIpAddress(address);
 +        }
 +    }
 +}
++
 +/**
 + * Initialize the private request filter by replacing the global HTTP and HTTPS agents with an instance of PrivateRequestAgent.
 + * @param {object} options Options for initializing the private request filter.
 + * @param {boolean} options.listen Whether the server is listening for incoming requests. This is used to determine whether to log a warning if the private request filter is not enabled.
 + * @param {boolean} options.enabled Whether the private request filter is enabled.
 + * @param {string[]} options.privateAddressWhitelist List of private IP addresses or CIDR ranges to allow.
 + * @param {boolean} options.logBlocked Whether to log blocked requests to the console.
 + * @param {boolean} options.logAllowed Whether to log allowed requests to the console.
 + * @param {boolean} options.allowUnresolvedHosts Whether to allow requests to hosts that cannot be resolved.
 + * @param {boolean} options.enableKeepAlive Whether to enable HTTP/HTTPS keep-alive.
 + */
 +export default function initPrivateRequestFilter({ listen, enabled, privateAddressWhitelist, logBlocked, logAllowed, allowUnresolvedHosts, enableKeepAlive }) {
 +    if (!enabled) {
 +        if (listen) {
 +            console.warn();
 +            console.warn(color.yellow('Warning: listen is enabled but private request filter is disabled. This may expose your server to SSRF attacks.'));
 +            console.warn(color.blue('To enable, provide trusted addresses in privateAddressWhitelist.allowedRanges and set privateAddressWhitelist.enabled to true in config.yaml and restart the server.'));
 +        }
 +        return;
 +    }
++
 +    const agent = new PrivateRequestAgent({ privateAddressWhitelist, logBlocked, logAllowed, allowUnresolvedHosts, enableKeepAlive });
++
 +    http.globalAgent = agent;
 +    https.globalAgent = agent;
++
 +    console.info();
 +    console.info(color.green(LOG_HEADER), 'Enabled');
 +    if (agent.privateAddressWhitelist.length > 0) {
 +        console.info(color.green(LOG_HEADER), 'Allowed private addresses:', color.blue(agent.privateAddressWhitelist.join(', ')));
 +    }
 +    console.info();
 +}

src/request-proxy.js+12 -3 Viewed

  * @property {string} url Proxy URL.
  * @property {string[]} bypass List of URLs to bypass proxy.
  * @property {boolean} enableKeepAlive Enable HTTP/HTTPS keep-alive.
 + * @property {boolean} privateRequestFilterEnabled Whether the private request filter is enabled.
  */
 export default function initRequestProxy({ enabled, url, bypass, enableKeepAlive, privateRequestFilterEnabled }) {
     try {
         // No proxy is enabled, so return
         if (!enabled) {
             return;
+        }
 +        if (privateRequestFilterEnabled) {
 +            console.warn(color.yellow(LOG_HEADER), 'Warning: Request proxy is enabled while private request filter is also enabled. Only URLs that BYPASS the request proxy will be checked.');
 +            console.warn(color.yellow(LOG_HEADER), 'To ensure all requests are properly filtered, disable the request proxy.');
 +        }
++
         if (!url) {
             console.error(color.red(LOG_HEADER), 'No proxy URL provided');
             return;

             process.env.no_proxy = bypass.join(',');
+        }
 -        const proxyAgentOptions =  enableKeepAlive ? { keepAlive: true } : { keepAlive: false };
 +        const httpAgent = http.globalAgent;
         const proxyAgenthttpsAgent = new ProxyAgent(proxyAgentOptions)https.globalAgent;
++
 +        const proxyAgent = new ProxyAgent({ httpAgent, httpsAgent, keepAlive: enableKeepAlive });
++
         http.globalAgent = proxyAgent;
         https.globalAgent = proxyAgent;

src/server-main.js+14 -1 Viewed

 import accessLoggerMiddleware, { getAccessLogPath, migrateAccessLog } from './middleware/accessLogWriter.js';
 import multerMonkeyPatch from './middleware/multerMonkeyPatch.js';
 import initRequestProxy from './request-proxy.js';
 +import initPrivateRequestFilter from './private-request-filter.js';
 import cacheBuster from './middleware/cacheBuster.js';
 import corsProxyMiddleware from './middleware/corsProxy.js';
 import hostWhitelistMiddleware from './middleware/hostWhitelist.js';

         exitProcess();
     });
 +    // Add private request filter.
 +    const requestFilterOptions = {
 +        listen: cliArgs.listen,
 +        enabled: !!getConfigValue('privateAddressWhitelist.enabled', false, 'boolean'),
 +        privateAddressWhitelist: getConfigValue('privateAddressWhitelist.allowedRanges', ['127.0.0.0/8', '::1/128']),
 +        logBlocked: !!getConfigValue('privateAddressWhitelist.log.blockedRequests', true, 'boolean'),
 +        logAllowed: !!getConfigValue('privateAddressWhitelist.log.allowedRequests', false, 'boolean'),
 +        allowUnresolvedHosts: !!getConfigValue('privateAddressWhitelist.allowUnresolvedHosts', false, 'boolean'),
 +        enableKeepAlive: cliArgs.enableKeepAlive,
 +    };
 +    initPrivateRequestFilter(requestFilterOptions);
++
     // Add request proxy.
     initRequestProxy({ enabled: cliArgs.requestProxyEnabled, url: cliArgs.requestProxyUrl, bypass: cliArgs.requestProxyBypass, enableKeepAlive: cliArgs.enableKeepAlive, privateRequestFilterEnabled: requestFilterOptions.enabled });
     // Wait for frontend libs to compile
     await webpackMiddleware.runWebpackCompiler({ pruneCache: true });

tests/private-request-filter.test.js+130 -0 Viewed

 +import { describe, test, expect, jest, beforeAll, beforeEach, afterAll } from '@jest/globals';
++
 +const mockNetConnect = jest.fn(() => ({ type: 'net-socket' }));
 +const mockTlsConnect = jest.fn(() => ({ type: 'tls-socket' }));
 +const mockLookup = jest.fn();
++
 +jest.unstable_mockModule('node:net', () => ({
 +    default: { connect: mockNetConnect },
 +}));
++
 +jest.unstable_mockModule('node:tls', () => ({
 +    default: { connect: mockTlsConnect },
 +}));
++
 +jest.unstable_mockModule('node:dns', () => ({
 +    default: { promises: { lookup: mockLookup } },
 +}));
++
 +jest.unstable_mockModule('../src/util.js', () => ({
 +    color: {
 +        red: text => text,
 +        green: text => text,
 +        blue: text => text,
 +        yellow: text => text,
 +    },
 +}));
++
 +jest.unstable_mockModule('../src/express-common.js', () => ({
 +    filterValidIpPatterns: patterns => patterns,
 +}));
++
 +/** @type {import('../src/private-request-filter.js').default} */
 +let initPrivateRequestFilter;
 +/** @type {import('node:http').default} */
 +let http;
 +/** @type {import('node:https').default} */
 +let https;
 +let originalHttpGlobalAgent;
 +let originalHttpsGlobalAgent;
++
 +beforeAll(async () => {
 +    ({ default: initPrivateRequestFilter } = await import('../src/private-request-filter.js'));
 +    ({ default: http } = await import('node:http'));
 +    ({ default: https } = await import('node:https'));
 +    originalHttpGlobalAgent = http.globalAgent;
 +    originalHttpsGlobalAgent = https.globalAgent;
 +});
++
 +beforeEach(() => {
 +    mockNetConnect.mockClear();
 +    mockTlsConnect.mockClear();
 +    mockLookup.mockReset();
 +    http.globalAgent = originalHttpGlobalAgent;
 +    https.globalAgent = originalHttpsGlobalAgent;
 +});
++
 +afterAll(() => {
 +    http.globalAgent = originalHttpGlobalAgent;
 +    https.globalAgent = originalHttpsGlobalAgent;
 +});
++
 +function initAgent({ privateAddressWhitelist = [], allowUnresolvedHosts = false } = {}) {
 +    initPrivateRequestFilter({
 +        listen: false,
 +        enabled: true,
 +        privateAddressWhitelist,
 +        logBlocked: false,
 +        logAllowed: false,
 +        allowUnresolvedHosts,
 +    });
++
 +    return http.globalAgent;
 +}
++
 +describe('private request filter', () => {
 +    test('allows direct private IP requests only when whitelisted', async () => {
 +        const agent = initAgent({ privateAddressWhitelist: ['127.0.0.0/8'] });
 +        await agent.connect({}, { host: '127.0.0.1', secureEndpoint: false });
++
 +        expect(mockNetConnect).toHaveBeenCalledWith(expect.objectContaining({ host: '127.0.0.1' }));
++
 +        const blockedAgent = initAgent({ privateAddressWhitelist: [] });
 +        await expect(blockedAgent.connect({}, { host: '127.0.0.1', secureEndpoint: false }))
 +            .rejects
 +            .toThrow('Blocked request to private IP address: 127.0.0.1');
 +    });
++
 +    test('resolves hostnames and blocks when DNS returns private IP', async () => {
 +        mockLookup.mockResolvedValue({ address: '192.168.1.8' });
 +        const agent = initAgent();
++
 +        await expect(agent.connect({}, { host: 'example.com', secureEndpoint: false }))
 +            .rejects
 +            .toThrow('Blocked request to private IP address: 192.168.1.8');
 +        expect(mockNetConnect).not.toHaveBeenCalled();
 +    });
++
 +    test('connects to resolved public IP to avoid hostname re-resolution', async () => {
 +        mockLookup.mockResolvedValue({ address: '93.184.216.34' });
 +        const agent = initAgent();
++
 +        await agent.connect({}, { host: 'example.com', secureEndpoint: false });
++
 +        expect(mockLookup).toHaveBeenCalledWith('example.com');
 +        expect(mockNetConnect).toHaveBeenCalledWith(expect.objectContaining({ host: '93.184.216.34' }));
 +    });
++
 +    test('handles unresolved hosts according to allowUnresolvedHosts setting', async () => {
 +        mockLookup.mockRejectedValue(new Error('lookup failed'));
 +        const blockedAgent = initAgent({ allowUnresolvedHosts: false });
++
 +        await expect(blockedAgent.connect({}, { host: 'missing-host.local', secureEndpoint: false }))
 +            .rejects
 +            .toThrow('Unable to resolve host: missing-host.local. Set privateAddressWhitelist.allowUnresolvedHosts to true to bypass this check.');
 +        expect(mockNetConnect).not.toHaveBeenCalled();
++
 +        const allowedAgent = initAgent({ allowUnresolvedHosts: true });
 +        await allowedAgent.connect({}, { host: 'missing-host.local', secureEndpoint: false });
 +        expect(mockNetConnect).toHaveBeenCalledWith(expect.objectContaining({ host: 'missing-host.local' }));
 +    });
++
 +    test('uses tls.connect for secure endpoints', async () => {
 +        mockLookup.mockResolvedValue({ address: '93.184.216.34' });
 +        const agent = initAgent();
 +        await agent.connect({}, { host: 'example.com', secureEndpoint: true });
 +        expect(mockLookup).toHaveBeenCalledWith('example.com');
 +        expect(mockTlsConnect).toHaveBeenCalledWith(expect.objectContaining({ host: '93.184.216.34' }));
 +        expect(mockNetConnect).not.toHaveBeenCalled();
 +    });
 +});