Kochbuch/recipes/fetch.py

"""Outgoing HTTP requests to user-supplied URLs, hardened against SSRF.

The host name is resolved once, every resolved address must be a public (global) IP, and
the connection is made to that checked IP (TLS still verifies the real host name). That
closes DNS-rebinding tricks. Redirects are followed manually and re-checked on every hop.
"""

import ipaddress
import socket
from dataclasses import dataclass
from urllib.parse import urljoin, urlsplit

import urllib3

# Honest identification as a bot; sites that refuse it are handled by the paste fallback.
USER_AGENT = "Mozilla/5.0 (compatible; Kochbuch-Rezeptimport/1.0; self-hosted recipe manager)"
MAX_REDIRECTS = 5
DEFAULT_MAX_BYTES = 3 * 1024 * 1024


class FetchError(Exception):
    def __init__(self, user_message):
        super().__init__(user_message)
        self.user_message = user_message


@dataclass
class FetchResult:
    url: str
    status: int
    content_type: str
    body: bytes

    @property
    def text(self):
        charset = "utf-8"
        for part in self.content_type.split(";")[1:]:
            key, _, value = part.strip().partition("=")
            if key.lower() == "charset" and value:
                charset = value.strip("\"'")
        try:
            return self.body.decode(charset, errors="replace")
        except LookupError:
            return self.body.decode("utf-8", errors="replace")


def _resolve_public(host, port):
    try:
        infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
    except (socket.gaierror, UnicodeError):
        raise FetchError("Die Adresse konnte nicht gefunden werden.") from None
    addresses = []
    for info in infos:
        ip = ipaddress.ip_address(info[4][0].split("%")[0])
        if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped:
            ip = ip.ipv4_mapped
        if not ip.is_global or ip.is_multicast:
            raise FetchError("Diese Adresse ist nicht erlaubt (internes Netz).")
        addresses.append(str(ip))
    if not addresses:
        raise FetchError("Die Adresse konnte nicht gefunden werden.")
    return addresses[0]


def _validate_url(url):
    parts = urlsplit(url)
    if parts.scheme not in ("http", "https") or not parts.hostname:
        raise FetchError("Bitte gib eine gültige http(s)-Adresse ein.")
    if parts.username or parts.password:
        raise FetchError("Adressen mit Zugangsdaten werden nicht unterstützt.")
    port = parts.port or (443 if parts.scheme == "https" else 80)
    if port not in (80, 443, 8080, 8443):
        raise FetchError("Dieser Port wird nicht unterstützt.")
    return parts, port


def _request_once(url, accept, timeout):
    parts, port = _validate_url(url)
    host = parts.hostname
    ip = _resolve_public(host, port)
    default_port = 443 if parts.scheme == "https" else 80
    headers = {
        "Host": host if port == default_port else f"{host}:{port}",
        "User-Agent": USER_AGENT,
        "Accept": accept,
        "Accept-Language": "de-DE,de;q=0.9,en;q=0.7",
    }
    if parts.scheme == "https":
        pool = urllib3.HTTPSConnectionPool(ip, port, server_hostname=host, assert_hostname=host,
                                           cert_reqs="CERT_REQUIRED", retries=False, timeout=timeout)
    else:
        pool = urllib3.HTTPConnectionPool(ip, port, retries=False, timeout=timeout)
    path = parts.path or "/"
    if parts.query:
        path += "?" + parts.query
    try:
        return pool.urlopen("GET", path, headers=headers, redirect=False, preload_content=False)
    except urllib3.exceptions.TimeoutError:
        raise FetchError("Die Seite hat nicht rechtzeitig geantwortet.") from None
    except urllib3.exceptions.SSLError:
        raise FetchError("Die sichere Verbindung zur Seite ist fehlgeschlagen.") from None
    except urllib3.exceptions.HTTPError:
        raise FetchError("Die Seite ist nicht erreichbar.") from None


def fetch(url, accept="text/html,application/xhtml+xml;q=0.9,*/*;q=0.8", max_bytes=DEFAULT_MAX_BYTES, timeout=None):
    timeout = timeout or urllib3.Timeout(connect=5, read=15)
    for _ in range(MAX_REDIRECTS + 1):
        response = _request_once(url, accept, timeout)
        try:
            if response.status in (301, 302, 303, 307, 308) and response.headers.get("Location"):
                url = urljoin(url, response.headers["Location"])
                continue
            chunks, size = [], 0
            for chunk in response.stream(64 * 1024, decode_content=True):
                size += len(chunk)
                if size > max_bytes:
                    raise FetchError("Die Seite ist zu groß.")
                chunks.append(chunk)
            return FetchResult(url=url, status=response.status,
                               content_type=response.headers.get("Content-Type", ""), body=b"".join(chunks))
        except urllib3.exceptions.HTTPError:
            raise FetchError("Die Seite konnte nicht vollständig geladen werden.") from None
        finally:
            response.release_conn()
    raise FetchError("Zu viele Weiterleitungen.")