Coverage for src/local_deep_research/security/dns_pinning.py: 91%
199 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Pin the validated IP for the actual outbound connection.
3Closes the resolve-vs-connect gap in the SSRF guard. ``validate_url``
4resolves a hostname via ``socket.getaddrinfo`` and checks the resulting
5addresses against the private/internal/metadata block lists, but
6``requests``/``urllib3`` then resolve the hostname *again, independently*
7at connect time. An attacker who controls a DNS authority can answer with
8a public address while the guard is looking and a private / cloud-metadata
9address a moment later at connect — so the address that was validated is
10not the address that gets connected to.
12This module makes the fetch connect to the SAME address that was
13validated. A thread-local registry of pinned ``host -> addrinfo`` entries
14is consulted by a process-wide ``socket.getaddrinfo`` shim installed once
15at import. While a host is pinned (for the duration of a single request,
16re-established at each redirect hop), the shim returns ONLY the
17pre-validated addresses for that host; every other lookup falls through to
18the real resolver untouched.
20Why a ``getaddrinfo`` shim rather than a custom ``HTTPAdapter``:
22* It covers BOTH sinks uniformly — the module-level ``requests.get`` /
23 ``requests.post`` used by ``safe_get`` / ``safe_post`` AND the
24 ``SafeSession.send`` path — without threading a mounted adapter through
25 every call site or per-request session.
26* TLS is preserved for free. The request URL keeps the original hostname,
27 so ``urllib3`` still sets the TLS SNI and verifies the server
28 certificate against that hostname. Only name resolution is redirected,
29 so there is NO certificate-verification bypass and no
30 ``server_hostname`` / ``assert_hostname`` plumbing to get wrong.
31* It behaves identically for ``http`` and ``https`` and re-pins cleanly at
32 every redirect hop.
33* Thread safety: the pin registry is thread-local, so concurrent requests
34 in LDR's thread pools never observe each other's pins, and every
35 unpinned lookup is an untouched pass-through. This mirrors the existing
36 ``security/egress`` ``socket.connect`` audit hook, which also installs a
37 process-wide, thread-local-gated interposition.
39The outbound-notification path (Apprise) reuses the same shim through
40:func:`pinned_notification_send`. Apprise fans out to arbitrary user URLs
41via ``requests``/``urllib3`` and re-resolves at send time — the exact
42resolve-vs-connect gap the shim closes — but it also follows redirects and
43can be pointed at a *new*, unpinned host. So the notification window pins
44every raw-webhook host in the batch AND activates a thread-local
45"block-private" mode (:func:`block_private_resolution`): for its duration,
46any UNPINNED lookup that resolves to a private/loopback/link-local/
47cloud-metadata address is refused at the ``getaddrinfo`` layer (a
48``socket.gaierror``), so a redirect-to-internal is blocked at the socket
49layer before a connection is made. Public unpinned hosts resolve normally.
50The block is thread-local, so it only affects the sending thread and can
51never wrongly block a legitimate private request running concurrently in
52another thread. It is activated only after Apprise is forced into
53synchronous, in-thread delivery (``AppriseAsset(async_mode=False)``); with
54the default async fan-out the send would run in worker threads that carry
55no thread-local pin/block and the guard would silently not apply.
57Import-order dependency: the pin only works while ``socket.getaddrinfo``
58is still THIS module's shim. If something replaces ``socket.getaddrinfo``
59wholesale AFTER this module is imported — e.g. ``gevent``/``eventlet``
60monkey-patching, or any other socket shim — that later patch silently
61overwrites ours and the pin is dropped fail-open (validation still runs,
62but the connect-time re-resolution it is meant to catch goes unguarded
63again). LDR runs Socket.IO in threading ``async_mode`` and does not
64monkeypatch sockets, so this does not currently apply — but if that ever
65changes, any such monkeypatching must happen BEFORE this module (or
66``security.safe_requests``, which imports it) is first imported.
67"""
69import contextlib
70import ipaddress
71import socket
72import threading
73from typing import Iterator, List, Optional, Tuple
75import requests
76from loguru import logger
77from urllib3.exceptions import LocationParseError
78from urllib3.util import parse_url
80from . import ssrf_validator
81from .log_sanitizer import redact_and_bound_for_log
83# Attribute stamped on our shim (see below) so any later capture of the
84# "real" resolver can recognize it and refuse to capture the shim as if it
85# were the genuine resolver.
86_SHIM_MARKER = "_ldr_dns_pin_shim"
89def _capture_real_getaddrinfo():
90 """Return the genuine resolver, never our own shim.
92 On a normal import ``socket.getaddrinfo`` is still the stdlib resolver
93 and is captured as-is. But if this module is *reloaded*
94 (``importlib.reload``) after :func:`install` has already swapped in our
95 shim, ``socket.getaddrinfo`` is that shim — capturing it here would make
96 the shim's pass-through call itself, recursing forever (``RecursionError``)
97 on every unpinned lookup process-wide. In that case fall back to the real
98 resolver captured by the previous load, which survives in this module's
99 globals across a reload.
100 """
101 current = socket.getaddrinfo
102 if not getattr(current, _SHIM_MARKER, False):
103 return current
104 # Reload after install(): don't re-capture our own shim. Reuse the real
105 # resolver the previous load already captured (guaranteed not to be a
106 # shim, thanks to this very guard).
107 previous = globals().get("_real_getaddrinfo")
108 if previous is not None and not getattr(previous, _SHIM_MARKER, False): 108 ↛ 110line 108 didn't jump to line 110 because the condition on line 108 was always true
109 return previous
110 return current
113# The genuine resolver, captured at import BEFORE the shim is installed.
114# Referenced by module-global name at call time (never closed over) so a
115# test can patch this single seam to control what BOTH the validation-time
116# resolution and the pin-time resolution observe. Captured via the guard
117# above so a reload cannot make the shim call itself.
118_real_getaddrinfo = _capture_real_getaddrinfo()
120# host (normalized) -> list of addrinfo 5-tuples that are safe to connect
121# to. Thread-local so concurrent fetches in different worker threads cannot
122# see one another's pins.
123#
124# The same thread-local also carries an optional ``block_private`` entry —
125# a ``(allow_localhost, allow_private_ips, block_link_local)`` tuple set by
126# :func:`block_private_resolution`. While present, every UNPINNED lookup on
127# this thread is resolved and re-checked against the SSRF policy, and a
128# result containing any blocked (private/loopback/link-local/metadata)
129# address is refused. See :func:`_resolve_maybe_block`.
130_thread_state = threading.local()
132# Schemes whose URL host is a real, resolvable connection target that the
133# client will connect to verbatim — so pinning the validated address is both
134# possible and worthwhile. These are the raw-webhook Apprise schemes
135# (``json``/``xml``/``form`` and their TLS variants) plus plain
136# ``http``/``https``.
137#
138# Every OTHER reachable Apprise scheme is deliberately excluded from pinning,
139# for one of two reasons — but both remaining kinds are still resolve-time
140# block-checked by the block-private window (:func:`block_private_resolution`),
141# so excluding them from pinning never leaves a cloud-metadata hole:
142#
143# * Token-host schemes (``discord://``, ``slack://``, ``tgram://`` …) put an
144# opaque credential/token in ``parse_url``'s host field, not a hostname.
145# Pinning would try to resolve that token as a name and break the send;
146# these plugins POST to their own hardcoded public API endpoints anyway.
147# * Self-hosted plugin schemes (``gotify://``, ``ntfy(s)://``, ``mattermost://``,
148# ``matrix://``, ``rocketchat://``, ``teams://`` …) DO carry a real,
149# resolvable host — the "opaque token" description does not apply to them.
150# They are left unpinned because their per-plugin URL→endpoint mapping is
151# plugin-specific (path/port rewriting) rather than a plain connect to the
152# URL host. The send-time block-private window enforces the selected
153# destination policy even when there is no precomputed pin.
154#
155# This block-window guarantee holds only for schemes whose network I/O resolves
156# through the shimmed ``socket.getaddrinfo`` — which every scheme reachable
157# today does (they send via ``requests``/``urllib3`` or ``smtplib``, both of
158# which call ``getaddrinfo``). Which schemes can reach this send path at all is
159# gated upstream by ``NotificationURLValidator.ALLOWED_SCHEMES``; a scheme that
160# connects via a primitive the shim cannot see (e.g. a raw ``socket.sendto``
161# datagram, as Apprise's ``rsyslog://`` uses) would bypass BOTH the pin and the
162# block window, so such a scheme must never be added to that allowlist.
163_PINNABLE_SCHEMES = frozenset(
164 {
165 "http",
166 "https",
167 "json",
168 "jsons",
169 "xml",
170 "xmls",
171 "form",
172 "forms",
173 }
174)
176_installed = False
179def _normalize_host(host: Optional[str]) -> Optional[str]:
180 """Normalize a host into the key form used by the pin registry.
182 Lower-cases (DNS is case-insensitive, and both ``urllib3.parse_url``
183 and ``urllib3``'s connection layer already lower-case the host before
184 it reaches ``getaddrinfo``), strips IPv6 brackets, and drops the
185 trailing root dot (``urllib3`` calls ``self._dns_host.rstrip(".")``
186 before resolving). Applying the same transform on both the pinning and
187 the lookup side guarantees the key matches.
188 """
189 if not host: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 return None
191 host = host.strip().lower()
192 if host.startswith("[") and host.endswith("]"): 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true
193 host = host[1:-1]
194 host = host.rstrip(".")
195 return host or None
198def _host_for_log(host) -> str:
199 """Best-effort text form of a host for a log message.
201 ``getaddrinfo`` accepts a ``bytes``/``bytearray`` host as well as a
202 ``str``; decode one to text so the block-window warning reads cleanly.
203 Decoding is for the log line only — the SSRF check runs on resolved IPs,
204 never on the host string — so it must never raise: ``surrogateescape``
205 round-trips any byte sequence. (The ``idna`` codec is deliberately NOT
206 used here: it does not support an error handler and raises on non-hostname
207 bytes.)
208 """
209 if isinstance(host, (bytes, bytearray)):
210 return bytes(host).decode("utf-8", errors="surrogateescape")
211 return str(host)
214def _get_pins() -> dict:
215 pins = getattr(_thread_state, "pins", None)
216 if pins is None:
217 pins = {}
218 _thread_state.pins = pins
219 return pins
222def reset_ssrf_block() -> None:
223 """Clear the per-thread "an SSRF block occurred" marker.
225 Called before a guarded ``notify()`` so :func:`consume_ssrf_block`
226 reflects only that attempt. See :func:`_resolve_maybe_block`.
227 """
228 _thread_state.ssrf_block_occurred = False
231def consume_ssrf_block() -> bool:
232 """Return whether the block-private window refused a lookup on this
233 thread since the last :func:`reset_ssrf_block`, and clear the marker.
235 Apprise swallows the ``socket.gaierror`` the block raises inside its
236 plugin, surfacing only a generic delivery failure. The sender
237 (``notifications.service._send_with_retry``) consults this after a failed
238 send to tell a confirmed security block apart from a transient error, so
239 the block can fail fast instead of being retried 3x.
240 """
241 occurred = getattr(_thread_state, "ssrf_block_occurred", False)
242 _thread_state.ssrf_block_occurred = False
243 return bool(occurred)
246# Send-time DNS resolution is bounded so a slow / hostile DNS authority
247# cannot hang the sending thread — which, on the notification path, is an
248# HTTP request handler (test_service is reached straight from the "Send Test
249# Notification" endpoint). Mirrors
250# ``notification_validator._resolve_hostname_ips``.
251_RESOLVE_TIMEOUT_SECONDS = 5
254def _getaddrinfo_bounded(host, port, family, type, proto, flags):
255 """Call the genuine resolver with a bounded timeout; fail closed.
257 Runs ``_real_getaddrinfo`` on a short-lived DAEMON thread (thread-safe
258 timeout, no ``socket.setdefaulttimeout`` process-global mutation) so a
259 hostile DNS authority that never answers cannot block the caller past
260 ``_RESOLVE_TIMEOUT_SECONDS``. On timeout a ``socket.gaierror`` is raised
261 — the lookup is REFUSED, never returned as a blank/partial answer a
262 caller might treat as "resolved" — so the timeout can only ever fail
263 closed. A resolution error raised by the resolver itself
264 (``socket.gaierror`` / ``OSError``) propagates unchanged.
266 The worker is a DAEMON thread deliberately: a ``getaddrinfo`` call that
267 hangs in C cannot be cancelled, and a non-daemon worker (as a
268 ``concurrent.futures`` pool thread is) would be joined by the interpreter
269 at shutdown — letting an in-flight hostile lookup block graceful process
270 exit / restart. A daemon worker is abandoned on timeout and never blocks
271 interpreter exit.
272 """
273 result: dict = {}
275 def _worker() -> None:
276 try:
277 result["value"] = _real_getaddrinfo(
278 host, port, family, type, proto, flags
279 )
280 except BaseException as exc: # noqa: BLE001 - propagated to caller
281 result["error"] = exc
283 worker = threading.Thread(
284 target=_worker,
285 name="ldr-dns-pin-resolve",
286 daemon=True,
287 )
288 worker.start()
289 worker.join(_RESOLVE_TIMEOUT_SECONDS)
290 if worker.is_alive():
291 # Timed out. The daemon worker is abandoned (it dies with the
292 # process) and the lookup is refused — fail closed.
293 raise socket.gaierror(
294 getattr(socket, "EAI_AGAIN", socket.EAI_FAIL),
295 f"DNS resolution for {host!r} timed out after "
296 f"{_RESOLVE_TIMEOUT_SECONDS}s during a pinned send (fail closed)",
297 )
298 if "error" in result:
299 # Re-raise the resolver's own error unchanged (e.g. gaierror for
300 # NXDOMAIN) so existing gaierror/OSError handling stays intact.
301 raise result["error"]
302 return result["value"]
305def _pinned_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
306 """Process-wide ``socket.getaddrinfo`` replacement.
308 Returns the pinned addresses when the calling thread has an active pin
309 for ``host``; otherwise delegates unchanged to the real resolver, so
310 this is a transparent pass-through for every lookup that is not part of
311 a pinned request.
312 """
313 key = _normalize_host(host) if isinstance(host, str) else None
314 if key is not None:
315 entry = _get_pins().get(key)
316 if entry is not None:
317 results = []
318 for fam, socktype, sockproto, canon, sockaddr in entry:
319 # Honor a caller-requested address family (AF_UNSPEC == 0
320 # means "any").
321 if family not in (0, socket.AF_UNSPEC, fam): 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 continue
323 # Honor a caller-requested socket type (0 == "any"). Every
324 # pinned entry is resolved as SOCK_STREAM (see
325 # _resolve_and_validate), which is also what urllib3's own
326 # connection lookups always request explicitly — so this
327 # never filters anything out of the normal HTTP(S) path. It
328 # only guards a hypothetical caller-requested SOCK_DGRAM (or
329 # other non-stream) lookup for a host that happens to be
330 # mid-pin, which would otherwise get back a SOCK_STREAM /
331 # IPPROTO_TCP entry mislabeled with the caller's requested
332 # type.
333 if type not in (0, socktype):
334 continue
335 # Rewrite the port from THIS lookup while preserving the
336 # IPv6 flowinfo / scope_id captured at pin time.
337 new_sockaddr = (sockaddr[0], port) + tuple(sockaddr[2:])
338 results.append(
339 (
340 fam,
341 type or socktype,
342 proto or sockproto,
343 canon,
344 new_sockaddr,
345 )
346 )
347 if results:
348 return results
349 # A pin exists but nothing matches the requested family/type.
350 # Fail closed rather than silently falling through to a fresh
351 # (attacker-controllable) resolution.
352 raise socket.gaierror(
353 getattr(socket, "EAI_ADDRFAMILY", socket.EAI_FAIL),
354 "No pinned address for the requested family/socktype",
355 )
356 # Not pinned. Delegate to the real resolver, but if a block-private
357 # window is active on this thread, refuse any answer that resolves to a
358 # blocked address (redirect-to-internal / rebind-to-metadata guard).
359 return _resolve_maybe_block(host, port, family, type, proto, flags)
362def _resolve_maybe_block(host, port, family, type, proto, flags):
363 """Real resolution for an unpinned host, honoring the block window.
365 When no block-private window is active this is a transparent
366 pass-through to the genuine resolver — with NO added timeout/thread
367 overhead, so ordinary process-wide DNS is completely unaffected by the
368 shim. When a window IS active (set by :func:`block_private_resolution`,
369 i.e. during a notification send), the resolution is (a) bounded by a
370 timeout so a slow / hostile DNS authority cannot hang the sending thread
371 (fail closed — a timeout refuses the lookup, it never returns a blank
372 answer), and (b) the resolved addresses are checked against the SSRF
373 policy and the lookup is refused with ``socket.gaierror`` if ANY of them
374 is blocked — closing a redirect-to-internal or rebind-to-metadata that
375 targets a host that was never pinned. The check runs on the SAME address
376 list the caller will connect to (this IS the ``getaddrinfo`` the client
377 uses), so there is no second resolution to race against.
379 No failure mode inside an active window escapes as a bare
380 ``ValueError``: an IDNA-unencodable host raises a native
381 ``UnicodeError`` (a ``ValueError`` subclass), which is caught below and
382 re-raised as ``socket.gaierror`` instead, since a ``ValueError``
383 reaching the sender would otherwise be mislabelled a confirmed
384 security block. A resolution error raised by the resolver itself
385 (``socket.gaierror`` / ``OSError``) propagates unchanged. The
386 IDNA case reaches this function through an allowed-but-not-pinnable scheme
387 (``ntfy``/``ntfys``, whose Apprise plugin parses with
388 ``verify_host=False``); see the inline comment below.
389 ``_resolve_and_validate`` enforces the same invariant at pin time.
390 """
391 block = getattr(_thread_state, "block_private", None)
392 if block is None:
393 # No window active: transparent, zero-overhead pass-through.
394 return _real_getaddrinfo(host, port, family, type, proto, flags)
395 # Window active — bound the resolution (send path) and fail closed on a
396 # timeout by letting the gaierror propagate to the client.
397 #
398 # ``socket.getaddrinfo`` raises ``UnicodeError`` (a ``ValueError``
399 # subclass) for a host that cannot be IDNA-encoded — a label over 63
400 # bytes, or an empty label. Letting that escape from inside an active
401 # window would break the same invariant ``_resolve_and_validate``
402 # protects one function away: at this layer a ``ValueError`` reaching
403 # the sender means a CONFIRMED SSRF block, so an unencodable name would
404 # be mislabelled as a security block (``SecurityBlockError``) instead
405 # of an ordinary resolution failure.
406 #
407 # This path IS reachable in practice, but not via a pin-time failure:
408 # ``pin_hosts`` only considers :data:`_PINNABLE_SCHEMES`, and for those
409 # Apprise's ``add()`` (``verify_host=True``) rejects an
410 # IDNA-unencodable host before a pin is ever attempted. The live route
411 # is a scheme that is ALLOWED but NOT pinnable — ``ntfy://`` /
412 # ``ntfys://``, whose Apprise plugin calls
413 # ``NotifyBase.parse_url(url, verify_host=False)`` and therefore
414 # accepts such a host. ``pin_hosts`` skips it on the scheme check, so
415 # it stays unpinned and its send-time lookup lands here, inside the
416 # active block-private window. Refuse it the way every other failed
417 # lookup in this shim is refused: ``socket.gaierror``, fail closed.
418 try:
419 results = _getaddrinfo_bounded(host, port, family, type, proto, flags)
420 except UnicodeError as exc:
421 raise socket.gaierror(
422 getattr(socket, "EAI_NONAME", getattr(socket, "EAI_FAIL", -1)),
423 f"DNS resolution for {_host_for_log(host)!r} failed: host is "
424 f"not encodable to IDNA (refused during a pinned send window)",
425 ) from exc
426 # A ``None`` host is an AF_PASSIVE bind
427 # (``getaddrinfo(None, port, ..., AI_PASSIVE)`` — a local listening-socket
428 # setup, not an outbound name to rebind), so it carries no attacker-
429 # controllable destination and is exempt from the block check. EVERY other
430 # host is checked, INCLUDING a ``bytes``/``bytearray`` host: ``getaddrinfo``
431 # accepts a bytes name and a bytes host (e.g. ``b"169.254.169.254"``)
432 # resolves to metadata/link-local exactly as a ``str`` host would, so
433 # returning it unchecked would be the one guard that fails OPEN. The check
434 # itself runs on the resolved IPs regardless of the host type — only the
435 # log line needs a text host, so a bytes host is decoded for display.
436 if host is None:
437 return results
438 host_display = _host_for_log(host)
439 allow_localhost, allow_private_ips, block_link_local = block
440 for info in results:
441 ip_str = str(info[4][0])
442 if ssrf_validator.is_ip_blocked(
443 ip_str,
444 allow_localhost=allow_localhost,
445 allow_private_ips=allow_private_ips,
446 block_link_local=block_link_local,
447 ):
448 logger.warning(
449 "Blocked send-time resolution of {} to internal/private/"
450 "metadata IP {} during the pinned notification window "
451 "(possible SSRF / redirect-to-internal)",
452 host_display,
453 ip_str,
454 )
455 # Mark the thread so the sender can tell this confirmed security
456 # block apart from a transient failure once Apprise has swallowed
457 # the gaierror below — a confirmed block must fail fast, not be
458 # retried. Consumed by ``consume_ssrf_block``.
459 _thread_state.ssrf_block_occurred = True
460 raise socket.gaierror(
461 getattr(socket, "EAI_FAIL", -1),
462 "Blocked resolution to an internal/private/metadata IP "
463 "during the pinned notification send window",
464 )
465 return results
468# Stamp our shim so a later capture (e.g. after ``importlib.reload``) can tell
469# it apart from the genuine resolver and refuse to capture it as the "real"
470# one — see :func:`_capture_real_getaddrinfo`.
471setattr(_pinned_getaddrinfo, _SHIM_MARKER, True)
474def install() -> None:
475 """Install the process-wide ``getaddrinfo`` shim (idempotent).
477 A no-op after the first call. The shim is transparent until a thread
478 activates a pin via :func:`pinned_request`, so importing this module
479 has no behavioral effect on code that does not fetch through the safe
480 request helpers.
482 Idempotent under module reload too: if a shim (this load's or a prior
483 load's) is already the active resolver, it is not stacked on top of
484 itself.
485 """
486 global _installed
487 if _installed: 487 ↛ 488line 487 didn't jump to line 488 because the condition on line 487 was never true
488 return
489 # Install our shim unless it is ALREADY EXACTLY this module's shim
490 # (identity check, not just the marker). A reload resets ``_installed`` to
491 # False and rebuilds ``_pinned_getaddrinfo`` as a fresh function object,
492 # while ``socket.getaddrinfo`` still points at the PREVIOUS load's shim —
493 # a stale, marker-stamped resolver whose thread-local reads live in the
494 # old module. Reinstalling here keeps ``socket.getaddrinfo`` in lockstep
495 # with the reloaded module so ``_shim_installed()`` — the invariant the
496 # notification send fails closed on — holds after a reload, instead of
497 # silently reporting the shim as missing forever.
498 #
499 # This cannot chain shim-onto-shim: ``_real_getaddrinfo`` is captured via
500 # ``_capture_real_getaddrinfo``, which never returns a shim, so the newly
501 # installed shim's pass-through calls the genuine resolver, not the old
502 # shim.
503 if socket.getaddrinfo is not _pinned_getaddrinfo: 503 ↛ 505line 503 didn't jump to line 505 because the condition on line 503 was always true
504 socket.getaddrinfo = _pinned_getaddrinfo
505 _installed = True
508def _shim_installed() -> bool:
509 """True iff THIS module's shim is still the active ``getaddrinfo``.
511 The pin and block-private guarantees only hold while
512 ``socket.getaddrinfo`` is exactly this module's ``_pinned_getaddrinfo``:
513 the thread-local pin/block state lives here, so a wholesale replacement
514 after import (``gevent``/``eventlet`` monkeypatch, another socket shim,
515 or a stale post-reload shim) would consult different state and silently
516 drop the guard. The check is strict identity — a *different* shim, even
517 one stamped with our marker, does not satisfy it because it would read a
518 different ``_thread_state``. Callers on the send path treat a False here
519 as fail-closed rather than sending unguarded.
520 """
521 return socket.getaddrinfo is _pinned_getaddrinfo
524def _extract_scheme_host(
525 url: Optional[str],
526) -> Tuple[Optional[str], Optional[str], bool]:
527 """Return ``(scheme, normalized_host, is_ip_literal)`` for ``url``.
529 ``scheme`` is lower-cased (``None`` when absent). ``host`` is ``None``
530 when the URL is empty or the parser rejects it. Uses
531 ``urllib3.util.parse_url`` — the same parser ``requests`` uses
532 internally and the same one ``ssrf_validator.validate_url`` validates
533 against — so the host we pin is the host the client will resolve.
534 """
535 if not url:
536 return None, None, False
537 try:
538 parsed = parse_url(url)
539 except (LocationParseError, ValueError):
540 return None, None, False
541 scheme = (parsed.scheme or "").lower() or None
542 host = _normalize_host(parsed.host)
543 if host is None: 543 ↛ 544line 543 didn't jump to line 544 because the condition on line 543 was never true
544 return scheme, None, False
545 try:
546 ipaddress.ip_address(host)
547 return scheme, host, True # IP literal — no DNS to rebind
548 except ValueError:
549 return scheme, host, False
552def _extract_host(url: Optional[str]) -> Tuple[Optional[str], bool]:
553 """Return ``(normalized_host, is_ip_literal)`` for ``url``.
555 Thin wrapper over :func:`_extract_scheme_host` for callers that don't
556 need the scheme (e.g. :func:`pinned_request`).
557 """
558 _scheme, host, is_literal = _extract_scheme_host(url)
559 return host, is_literal
562def _resolve_and_validate(
563 host: str,
564 allow_localhost: bool,
565 allow_private_ips: bool,
566 block_link_local: bool = False,
567) -> List[tuple]:
568 """Resolve ``host`` once and validate every returned address.
570 Uses the real resolver (never the shim, to avoid recursion and to
571 ignore any pin) and applies exactly the ``is_ip_blocked`` policy that
572 ``validate_url`` applies — honoring ``allow_localhost`` /
573 ``allow_private_ips`` (and ``block_link_local`` for the notification
574 path) while the ``ALWAYS_BLOCKED_METADATA_IPS`` set keeps firing
575 regardless. This is the connect-time re-validation that the pin adds: an
576 answer that changed since ``validate_url`` ran is caught here.
578 Returns the addrinfo list to pin (all entries validated safe).
580 Raises:
581 ValueError: if any resolved address is blocked. SSRF-style message
582 so existing callers/tests catch it uniformly. NOTE: this is
583 raised ONLY for a confirmed SSRF block (below) — a resolver
584 failure is always routed through ``requests.ConnectionError``
585 instead, specifically so ``send()``'s ``except ValueError``
586 handler (which reports a confirmed security block) is never
587 reached by an ordinary resolution failure.
588 requests.ConnectionError: if the host no longer resolves at pin
589 time, if resolution times out (a slow / hostile DNS authority
590 cannot hang the send), OR if the host is unencodable to IDNA
591 (``socket.getaddrinfo`` raises ``UnicodeError`` — a
592 ``UnicodeEncodeError`` is a ``ValueError`` subclass — for a
593 label over 63 bytes or an empty label). This particular catch
594 is currently unreachable in practice: it runs only for a
595 scheme in :data:`_PINNABLE_SCHEMES`, and those Apprise plugins
596 parse with ``verify_host=True``, so ``Apprise.add()`` rejects
597 an IDNA-unencodable host before ``_dispatch`` ever calls
598 ``pin_hosts``. The catch keeps the
599 ValueError-means-SSRF-block invariant true at THIS layer
600 rather than relying on that upstream reject. (The SEND-time
601 sibling in :func:`_resolve_maybe_block` is genuinely
602 reachable — see its comment — via a non-pinnable scheme.)
603 Fail closed as an ordinary transport error so the existing
604 retry / ``RequestException`` handling stays intact.
605 """
606 try:
607 addr_info = _getaddrinfo_bounded(
608 host, None, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, 0
609 )
610 except (socket.gaierror, UnicodeError) as exc:
611 raise requests.ConnectionError(
612 f"DNS resolution failed while pinning host {host}"
613 ) from exc
614 if not addr_info: 614 ↛ 615line 614 didn't jump to line 615 because the condition on line 614 was never true
615 raise requests.ConnectionError(
616 f"No addresses resolved while pinning host {host}"
617 )
619 for info in addr_info:
620 ip_str = str(info[4][0])
621 if ssrf_validator.is_ip_blocked(
622 ip_str,
623 allow_localhost=allow_localhost,
624 allow_private_ips=allow_private_ips,
625 block_link_local=block_link_local,
626 ):
627 logger.warning(
628 "Blocked connect-time address for host {}: resolves to "
629 "internal/private/metadata IP {} "
630 "(resolve-vs-connect guard)",
631 host,
632 ip_str,
633 )
634 raise ValueError(
635 "URL failed security validation (possible SSRF): "
636 f"host {host} resolved to blocked IP {ip_str} at connect time"
637 )
638 return addr_info
641@contextlib.contextmanager
642def pinned_request(
643 url: str,
644 allow_localhost: bool = False,
645 allow_private_ips: bool = False,
646 block_link_local: bool = False,
647) -> Iterator[Optional[List[tuple]]]:
648 """Pin DNS for the duration of a single outbound request to ``url``.
650 Resolves the URL host ONCE, validates every returned address against
651 the SSRF policy, and pins those addresses so the connection made by
652 ``requests`` / ``urllib3`` inside this context cannot be steered to a
653 rebind target: the address validated here is the address connected to.
655 ``block_link_local`` is forwarded to the connect-time validation so a
656 caller that keeps link-local blocked under ``allow_private_ips`` (e.g.
657 ``SafeSession(block_link_local=True)``) is not undone by a resolver
658 answer that moved into ``169.254.0.0/16`` between validation and
659 connect.
661 Yields the validated address list so the caller can apply any run-level
662 egress scope to the exact addresses that will be connected to. Yields
663 ``None`` without pinning when the host is an IP literal (no DNS to
664 rebind — the caller's ``validate_url`` already vetted the literal) or
665 when the URL host cannot be extracted.
667 Must wrap every outbound request AND every redirect hop, so each hop
668 re-resolves, re-validates, and re-pins.
670 Raises:
671 ValueError: if the host resolves to a blocked address at connect
672 time.
673 requests.ConnectionError: if the host fails to resolve at pin time.
674 """
675 host, is_literal = _extract_host(url)
676 if host is None or is_literal:
677 yield None
678 return
680 entry = _resolve_and_validate(
681 host,
682 allow_localhost,
683 allow_private_ips,
684 block_link_local=block_link_local,
685 )
687 pins = _get_pins()
688 # Save/restore any prior pin for this host so nested contexts (e.g. a
689 # redirect chain that loops back to the same host) are safe.
690 had_previous = host in pins
691 previous = pins.get(host)
692 pins[host] = entry
693 try:
694 yield entry
695 finally:
696 if had_previous: 696 ↛ 697line 696 didn't jump to line 697 because the condition on line 696 was never true
697 pins[host] = previous
698 else:
699 pins.pop(host, None)
702@contextlib.contextmanager
703def block_private_resolution(
704 allow_localhost: bool = False,
705 allow_private_ips: bool = False,
706 block_link_local: bool = False,
707) -> Iterator[None]:
708 """Refuse UNPINNED lookups that resolve to a blocked address.
710 While this context is active on the calling thread, any host that is
711 NOT covered by an active pin and resolves to a private / loopback /
712 link-local / cloud-metadata address (per ``ssrf_validator.is_ip_blocked``
713 with the supplied flags) is refused at the ``getaddrinfo`` layer with a
714 ``socket.gaierror``. Pinned hosts are unaffected (they short-circuit to
715 their validated addresses), and public unpinned hosts resolve normally.
717 ``block_link_local`` (every notification partition): when True, the
718 whole link-local range stays refused even under ``allow_private_ips=True``
719 — closing metadata reachable in link-local beyond the always-blocked
720 literals (e.g. Scaleway ``169.254.42.42``) without over-blocking RFC1918 /
721 loopback / non-link-local ULA self-hosted notifiers.
723 This closes redirect-to-internal / rebind-to-metadata for hosts that
724 were never pinned (e.g. a webhook that 302-redirects to
725 ``169.254.169.254`` or ``127.0.0.1``): the block runs on the very
726 resolution the client is about to connect to, so there is no window to
727 race.
729 Thread-local: only the calling thread is affected, so a legitimate
730 private request in another thread is never disturbed. Windows nest
731 (inner replaces outer, restored on exit).
732 """
733 prev = getattr(_thread_state, "block_private", None)
734 _thread_state.block_private = (
735 allow_localhost,
736 allow_private_ips,
737 block_link_local,
738 )
739 try:
740 yield
741 finally:
742 _thread_state.block_private = prev
745@contextlib.contextmanager
746def pin_hosts(
747 urls,
748 allow_localhost: bool = False,
749 allow_private_ips: bool = False,
750 block_link_local: bool = False,
751) -> Iterator[None]:
752 """Pin the validated address of every pinnable host in ``urls``.
754 For each URL whose scheme is in :data:`_PINNABLE_SCHEMES` and whose host
755 is a real (non-literal) name, resolves once, validates every returned
756 address against the SSRF policy, and pins those addresses for the
757 duration of the context. IP literals, token-host plugin schemes, and
758 unparseable URLs are skipped (nothing to rebind, or the host is not a
759 resolvable name). A host that fails to resolve right now is also skipped
760 rather than aborting the whole batch — it is left unpinned so the
761 accompanying block-private window still governs its send-time lookup and
762 Apprise fails only that one URL.
764 ``block_link_local`` is forwarded to the connect-time validation so a
765 raw-webhook host that resolves into link-local (metadata territory) is
766 refused even under ``allow_private_ips=True`` (notification lenient
767 partition).
769 Raises:
770 ValueError: if a pinnable host resolves to a blocked address (the
771 batch is refused — fail closed).
772 """
773 pins = _get_pins()
774 saved: List[Tuple[str, bool, Optional[list]]] = []
775 pinned_now = set()
776 try:
777 for url in urls:
778 scheme, host, is_literal = _extract_scheme_host(url)
779 if scheme not in _PINNABLE_SCHEMES:
780 continue
781 if host is None or is_literal or host in pinned_now: 781 ↛ 782line 781 didn't jump to line 782 because the condition on line 781 was never true
782 continue
783 try:
784 entry = _resolve_and_validate(
785 host,
786 allow_localhost,
787 allow_private_ips,
788 block_link_local=block_link_local,
789 )
790 except requests.ConnectionError:
791 # Skipping leaves the host unpinned, so its send-time
792 # lookup goes through ``_resolve_maybe_block``, which
793 # applies the same policy inside the block window. This
794 # nominally includes the IDNA-unencodable host
795 # (``UnicodeError`` routed to ``ConnectionError`` by
796 # ``_resolve_and_validate``), though that shape does not
797 # reach here today: for a pinnable scheme Apprise's
798 # ``add()`` rejects such a host first. The reachable
799 # IDNA route into ``_resolve_maybe_block`` is a scheme
800 # skipped by the ``_PINNABLE_SCHEMES`` check above.
801 logger.opt(lazy=True).debug(
802 "Skipping pin for currently-unresolvable host {}",
803 # (#6938: a request-supplied host is unbounded;
804 # redacted only when a DEBUG sink takes the line.)
805 lambda: redact_and_bound_for_log(host, 256),
806 )
807 continue
808 saved.append((host, host in pins, pins.get(host)))
809 pins[host] = entry
810 pinned_now.add(host)
811 yield
812 finally:
813 for host, had_previous, previous in reversed(saved):
814 if had_previous: 814 ↛ 815line 814 didn't jump to line 815 because the condition on line 814 was never true
815 pins[host] = previous
816 else:
817 pins.pop(host, None)
820class NotificationGuardUnavailableError(RuntimeError):
821 """The DNS-pin shim required for a guarded notification send is not
822 installed as the active ``socket.getaddrinfo`` resolver.
824 Raised ONLY by :func:`pinned_notification_send` (never by
825 :func:`pinned_request`, the general-purpose safe_get/safe_post guard,
826 which does not depend on this pre-check). This is a deliberate
827 fail-closed SECURITY REFUSAL, not an incidental runtime error: the
828 pin/block-private window cannot be guaranteed without the shim, so the
829 send is refused outright.
831 Subclasses ``RuntimeError`` (not just ``Exception``) so any code that
832 still generically catches ``RuntimeError`` around a guarded send
833 continues to behave the same. It is a dedicated subclass — rather than
834 a bare ``RuntimeError`` — so ``notifications.service`` can catch this
835 specific refusal and re-raise it as its own ``SecurityBlockError``
836 (non-retryable, INVALID_URL classification) without this general
837 security module importing exception types from the higher-level
838 ``notifications`` package (which would invert the module layering and
839 risk a circular import — ``notifications.service`` imports this
840 module).
841 """
844@contextlib.contextmanager
845def pinned_notification_send(
846 urls,
847 allow_localhost: bool = False,
848 allow_private_ips: bool = False,
849 block_link_local: bool = False,
850) -> Iterator[None]:
851 """Guard a single in-thread Apprise ``notify()`` over ``urls``.
853 Combines :func:`pin_hosts` (pin every raw-webhook host in the batch to
854 its validated address) with :func:`block_private_resolution` (refuse any
855 unpinned lookup — a redirect target or a token-scheme endpoint — that
856 resolves to a blocked address). Both are thread-local, so the caller
857 MUST run the ``notify()`` synchronously in this thread
858 (``AppriseAsset(async_mode=False)``); an async fan-out would resolve in
859 worker threads that carry neither the pin nor the block.
861 ``allow_private_ips`` is the operator flag for every notification
862 partition (HTTP, host-bearing plugins and fixed vendor endpoints). It
863 is public-only by default; with the opt-in, private targets and a
864 private forward proxy (``HTTPS_PROXY``) are allowed.
866 ``block_link_local`` is True for every notification partition so the
867 whole link-local range stays blocked even when private LAN addresses
868 are operator-approved. It applies to both pin and send-time resolution.
870 Raises:
871 NotificationGuardUnavailableError: if the ``getaddrinfo`` shim is not
872 the active resolver (see :func:`_shim_installed`). The
873 pin/block cannot be guaranteed in that state, so the send is
874 REFUSED (fail closed) rather than proceeding unguarded. A
875 ``RuntimeError`` subclass — see its docstring for why it is
876 not a bare ``RuntimeError``.
877 """
878 # Fail closed if our shim is not installed: without it the pin registry
879 # and the block-private window are never consulted, so proceeding would
880 # send completely unguarded (silent fail-open). Refuse instead.
881 if not _shim_installed():
882 raise NotificationGuardUnavailableError(
883 "DNS pin shim is not the active socket.getaddrinfo; refusing the "
884 "notification send (fail closed). The resolve-vs-connect pin and "
885 "block-private window cannot be guaranteed — see "
886 "security.dns_pinning."
887 )
888 with contextlib.ExitStack() as stack:
889 stack.enter_context(
890 pin_hosts(
891 urls,
892 allow_localhost,
893 allow_private_ips,
894 block_link_local=block_link_local,
895 )
896 )
897 stack.enter_context(
898 block_private_resolution(
899 allow_localhost,
900 allow_private_ips,
901 block_link_local=block_link_local,
902 )
903 )
904 yield
907# Install on import: safe_requests imports this module, so the shim is in
908# place before any fetch. Transparent until a pin is active.
909install()