Coverage for src/local_deep_research/utilities/url_utils.py: 95%
163 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""URL utility functions for the local deep research application."""
3from functools import lru_cache
4from typing import Optional
5import re
6from urllib.parse import parse_qsl, unquote, urlencode, urlsplit, urlunsplit
8from loguru import logger
10from .chunk_anchor import (
11 LIBRARY_ALIAS_HOST,
12 LIBRARY_ROUTE_PREFIXES,
13 LIBRARY_ROUTE_SUFFIX_ALTERNATION,
14 MAX_CHUNK_INDEX,
15 is_safe_document_id,
16)
17from ..security import redact_url_for_log, validate_url
18from ..security.network_utils import is_private_ip
20# Re-export for backwards compatibility
21__all__ = [
22 "normalize_url",
23 "is_private_ip",
24 "canonical_url_key",
25 "library_display_url",
26 "is_safe_custom_llm_endpoint",
27]
29# Tracking query parameter keys (matched lowercased).
30_TRACKING_PARAMS = frozenset(
31 {
32 "fbclid",
33 "gclid",
34 "msclkid",
35 "yclid",
36 "dclid",
37 "gad_source",
38 "mc_eid",
39 "mc_cid",
40 "ref_src",
41 "igshid",
42 "_ga",
43 "_gl",
44 }
45)
46# Tracking param name prefixes (matched lowercased).
47_TRACKING_PREFIXES = ("utm_",)
50def normalize_url(raw_url: str) -> str:
51 """
52 Normalize a URL to ensure it has a proper scheme and format.
54 Args:
55 raw_url: The raw URL string to normalize
57 Returns:
58 A properly formatted URL string
60 Examples:
61 >>> normalize_url("localhost:11434")
62 'http://localhost:11434'
63 >>> normalize_url("https://example.com:11434")
64 'https://example.com:11434'
65 >>> normalize_url("http:example.com")
66 'http://example.com'
67 """
68 if not raw_url:
69 raise ValueError("URL cannot be empty")
71 # Clean up the URL
72 raw_url = raw_url.strip()
74 # First check if the URL already has a proper scheme
75 if raw_url.startswith(("http://", "https://")):
76 return raw_url
78 # Handle case where URL is malformed like "http:hostname" (missing //)
79 if raw_url.startswith(("http:", "https:")) and not raw_url.startswith(
80 ("http://", "https://")
81 ):
82 scheme = raw_url.split(":", 1)[0]
83 rest = raw_url.split(":", 1)[1]
84 return f"{scheme}://{rest}"
86 # Handle URLs that start with //
87 if raw_url.startswith("//"):
88 # Remove the // and process
89 raw_url = raw_url[2:]
91 # At this point, we should have hostname:port or just hostname
92 # Determine if this is localhost or an external host
93 hostname = raw_url.split(":")[0].split("/")[0]
95 # Handle IPv6 addresses in brackets
96 if hostname.startswith("[") and "]" in raw_url:
97 # Extract the IPv6 address including brackets
98 hostname = raw_url.split("]")[0] + "]"
100 # Use http for local/private addresses, https for external hosts
101 scheme = "http" if is_private_ip(hostname) else "https"
103 return f"{scheme}://{raw_url}"
106# Internal library-document routes emitted as citation URLs by the RAG
107# search engines (see ``LibraryRAGSearchEngine._get_document_url``). These
108# are the only scheme-less URLs whose shape ``canonical_url_key`` knows
109# well enough to canonicalize; everything else falls back to the raw
110# string. Imported from ``chunk_anchor`` so the producer side (which
111# decides what counts as a library hit) and this consumer side cannot
112# disagree about the prefix set.
113_LIBRARY_ROUTE_PREFIXES = LIBRARY_ROUTE_PREFIXES
116# The absolute alias the agent sometimes emits for a library document; the
117# fetch resolver accepts it (see ``library_resolver._parse_library_reference``)
118# and hands the ORIGINAL string back, which is then registered as a citation
119# link. It must key and display as the relative route it denotes, or one
120# document occupies two bibliography entries.
121_LIBRARY_ALIAS_SCHEME = "https"
122_LIBRARY_ALIAS_HOST = LIBRARY_ALIAS_HOST
123# Mirrors ``library_resolver._LIBRARY_PATH_RE``: a document id plus at most
124# one known view. Anything deeper is not a library route and must not be
125# rewritten into one.
126_LIBRARY_ALIAS_PATH_RE = re.compile(
127 rf"^/(?P<doc_id>[^/?#]+)(?:/(?P<suffix>{LIBRARY_ROUTE_SUFFIX_ALTERNATION}))?/?$"
128)
129# The same shape for the RELATIVE routes, including the ``/lib/`` spelling.
130# Both spellings denote one document (``library_resolver._LIBRARY_PATH_RE``
131# accepts either), so both normalize to the ``/library/`` form: keying them
132# separately fans a document across two bibliography entries, and only
133# ``/library/document/<id>`` is a registered Flask route — a ``/lib/...``
134# link renders dead.
135_LIBRARY_ROUTE_PATH_RE = re.compile(
136 r"^/(?:library|lib)/document/(?P<doc_id>[^/?#]+)"
137 rf"(?:/(?P<suffix>{LIBRARY_ROUTE_SUFFIX_ALTERNATION}))?/?$"
138)
141def _is_library_route(path: str) -> bool:
142 return path.startswith(_LIBRARY_ROUTE_PREFIXES)
145def _has_control_chars(value: str) -> bool:
146 """True if *value* carries a C0/DEL character.
148 These are rendered verbatim into the ``## Sources`` block, where an
149 embedded newline forges extra lines. The library-route branches build
150 their result by string-splitting rather than via ``urlunsplit``, so —
151 unlike the absolute-URL branch — they cannot drop such a character on
152 their own and must reject it explicitly.
153 """
154 return any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in value)
157def _normalize_library_alias(url: str) -> str | None:
158 """Map ``https://library.document/<id>[/pdf|/chunks]`` to its relative route.
160 Returns ``None`` for anything else. The accepted shape is deliberately
161 narrow — a document id followed by at most the ``pdf`` or ``chunks``
162 view, mirroring ``library_resolver._LIBRARY_PATH_RE`` — because the
163 result is keyed AND displayed as an internal route. A permissive
164 rewrite lets a crafted alias merge into a real document's bibliography
165 entry, inherit its title, and replace its link: pasting an unvalidated
166 path after ``/library/document`` turns
167 ``https://library.document/<id>/../../../admin/x#chunk-1`` into a
168 same-origin link to ``/admin/x`` rendered under the real document's
169 name.
171 Two deliberate differences from the resolver, both narrowing or
172 identity-preserving rather than widening:
174 * A query disqualifies (as in the resolver) — it can carry
175 server-meaningful state, unlike a client-side anchor.
176 * Host comparison uses ``hostname``/``port``, so ``LIBRARY.DOCUMENT``,
177 ``:443`` and userinfo forms name the same document rather than
178 fanning out into separate bibliography entries. The resolver rejects
179 those for fetching; they are still the same source for citation
180 purposes.
182 The fragment is carried over — it is the chunk anchor.
183 """
184 # Cheap reject first: this runs once per citation inside
185 # ``format_links_to_markdown``, and almost every real citation is a
186 # plain external URL that can never match. Skipping ``urlsplit`` for
187 # those keeps the bibliography pass off a per-link parse.
188 if "://" not in url or _LIBRARY_ALIAS_HOST not in url.lower():
189 return None
190 # Reject tab/newline/CR before ``urlsplit`` sees them: it deletes them
191 # silently, which would collide ``/a<TAB>b`` with a real ``/ab`` and
192 # rewrite the citation to a DIFFERENT document. The neighbouring
193 # manual splits avoid ``urlsplit`` for exactly this reason.
194 if any(ch in url for ch in "\t\n\r"): 194 ↛ 195line 194 didn't jump to line 195 because the condition on line 194 was never true
195 return None
196 try:
197 parsed = urlsplit(url)
198 except ValueError:
199 return None
200 if (
201 parsed.scheme != _LIBRARY_ALIAS_SCHEME
202 or parsed.query
203 or (parsed.hostname or "") != _LIBRARY_ALIAS_HOST
204 ):
205 return None
206 try:
207 if parsed.port not in (None, 443):
208 return None
209 except ValueError: # malformed port
210 return None
211 path_match = _LIBRARY_ALIAS_PATH_RE.match(parsed.path)
212 if not path_match:
213 return None
214 doc_id = unquote(path_match.group("doc_id"))
215 # Same predicate ``extract_document_id`` applies, imported rather than
216 # restated: the doc id is decoded before BOTH validating and
217 # re-emitting, so ``/%2D`` and ``/-`` forms of one id give one key.
218 if not is_safe_document_id(doc_id): 218 ↛ 219line 218 didn't jump to line 219 because the condition on line 218 was never true
219 return None
220 suffix = path_match.group("suffix")
221 rel = f"/library/document/{doc_id}"
222 if suffix:
223 rel = f"{rel}/{suffix}"
224 return f"{rel}#{parsed.fragment}" if parsed.fragment else rel
227def _parse_library_citation(raw: str) -> tuple[str, str] | None:
228 """Return ``(dedup_key, display_url)`` for a library citation, else ``None``.
230 One parser for every accepted spelling — the relative
231 ``/library/document/<id>`` route, its ``/lib/`` abbreviation, and the
232 absolute ``https://library.document/<id>`` alias — so the two paths
233 cannot enforce different rules. They used to: only the alias validated
234 its document id, which meant the exact traversal this function's
235 sibling docstring promises to block sailed straight through when the
236 same URL arrived relative. ``/library/document/7/../../../admin/wipe``
237 keyed as document ``7`` and *displayed* as ``/admin/wipe``, so a
238 crafted citation merged into a real document's entry, inherited its
239 title and citation numbers, and replaced its link.
241 * ``dedup_key`` is ``/library/document/<id>`` — the ``/pdf``,
242 ``/chunks`` and ``#chunk-<n>`` views of one document are one source
243 and belong on one bibliography line.
244 * ``display_url`` keeps the view suffix and the ``#chunk-<n>`` anchor,
245 which is what makes a citation scroll to the cited text.
247 Normalisations, all identity-preserving: ``/lib/`` becomes
248 ``/library/``, the document id is percent-decoded (so ``a%2Db`` and
249 ``a-b`` are one key, matching what the resolver looks up), and a query
250 string is dropped (the route ignores it).
251 """
252 if not raw:
253 return None
254 stripped = raw.strip()
255 # Cheap reject before any parsing: the overwhelming majority of
256 # citations are ordinary external URLs.
257 if not (
258 stripped.startswith(_LIBRARY_ROUTE_PREFIXES)
259 or ("://" in stripped and _LIBRARY_ALIAS_HOST in stripped.lower())
260 ):
261 return None
262 # Refuse anything carrying a control character rather than truncating
263 # the doc-id segment at it. Truncating keys the crafted URL as the
264 # real ``/library/document/<id>`` it was prefixed with, merging it
265 # into that document's bibliography entry and lending it the
266 # document's title and citation number. Returning ``None`` keeps the
267 # two sources apart; the payload still cannot forge a Sources line,
268 # because every field rendered into that block goes through
269 # ``search_utilities._sanitize_sources_field``.
270 if _has_control_chars(stripped):
271 return None
272 alias = _normalize_library_alias(stripped)
273 if alias is not None:
274 stripped = alias
275 if not _is_library_route(stripped):
276 return None
277 # Split fragment and query manually rather than via ``urlsplit``,
278 # which silently deletes embedded tab/newline and would collide
279 # ``/a<TAB>b`` with a real ``/ab``.
280 head, sep, fragment = stripped.partition("#")
281 path = head.split("?", 1)[0]
282 path_match = _LIBRARY_ROUTE_PATH_RE.match(path)
283 if not path_match:
284 return None
285 doc_id = unquote(path_match.group("doc_id"))
286 # Same predicate ``extract_document_id`` applies, imported rather than
287 # restated: the id is decoded before BOTH validating and re-emitting,
288 # so ``/%2D`` and ``/-`` forms of one id give one key.
289 if not is_safe_document_id(doc_id):
290 return None
291 key = f"/library/document/{doc_id}"
292 suffix = path_match.group("suffix")
293 display = f"{key}/{suffix}" if suffix else key
294 # Re-attach the fragment ONLY when it is a real chunk anchor. A
295 # library route carries no other meaningful fragment, so anything else
296 # is either a producer bug or an unvalidated string that reached us
297 # from the agent (``fetch_content`` passes its argument through
298 # ``library_resolver`` verbatim). Re-emitting it built a citation URL
299 # whose anchor cannot name a chunk — a dead link that still claims to
300 # cite one — and smuggled arbitrary text into the rendered Sources
301 # block, bypassing the ``MAX_CHUNK_INDEX``/format contract that
302 # ``chunk_anchor`` calls itself the single source of truth for.
303 # Dropping it keeps the citation (the route still resolves to the
304 # document) and loses only an anchor that never pointed anywhere.
305 if sep and fragment and is_valid_chunk_fragment(fragment):
306 display = f"{display}#{fragment}"
307 return key, display
310def library_display_url(raw: str) -> str | None:
311 """Return the display URL when *raw* is a library route, else ``None``.
313 ``canonical_url_key`` collapses library routes to a per-document key,
314 which is right for grouping and wrong for display: the ``#chunk-<n>``
315 anchor is the entire point of a chunk-targeted citation. Renderers
316 group on the key and display this.
318 The returned URL is the *normalized* route (``/library/`` spelling,
319 percent-decoded id, no query), never the caller's raw string — a
320 string that does not parse as a library route returns ``None`` rather
321 than being echoed back into the report.
322 """
323 parsed = _parse_library_citation(raw)
324 return parsed[1] if parsed is not None else None
327# A chunk anchor is ``#chunk-<n>`` with the exact index shape
328# ``build_chunk_anchor_url`` emits. ``\d`` is Unicode-wide, so it is spelled
329# out here: ``chunk-\u0667`` and ``chunk-１２`` would otherwise be accepted
330# while ``chunk_anchor.extract_chunk_index`` — which calls itself the
331# single source of truth for this — rejects them.
332# ``\Z``, not ``$``: Python's ``$`` also matches just before a single
333# trailing newline even without ``re.MULTILINE``, so ``chunk-1\n``
334# passed. Both current callers strip and reject control characters
335# upstream, so nothing reaches here with one today — but this is the
336# shared predicate, and a future caller that skips those gates would
337# inherit a newline-smuggling hole in the one rule both paths trust.
338_CHUNK_FRAGMENT_RE = re.compile(r"^chunk-(0|[1-9][0-9]*)\Z", re.ASCII)
341def is_valid_chunk_fragment(fragment: str) -> bool:
342 """Return ``True`` if *fragment* (the part after ``#``) is a chunk
343 anchor that can actually name a chunk.
345 Shared by :func:`library_display_url` (which drops a fragment failing
346 this test) and :func:`preferred_chunk_display` (which rejects the whole
347 URL), so the render path and the store path cannot disagree about what
348 a valid anchor is.
349 """
350 if not isinstance(fragment, str): 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true
351 return False
352 match = _CHUNK_FRAGMENT_RE.match(fragment)
353 if match is None:
354 return False
355 # Bound the digit run BEFORE converting: the pattern admits an
356 # unbounded one, and int() raises above CPython's 4300-digit limit —
357 # out of a pure formatter with no caller catching it, which would take
358 # down the whole Sources block.
359 if len(match.group(1)) > len(str(MAX_CHUNK_INDEX)):
360 return False
361 # Bound it the way the producer does, so a fragment that cannot name a
362 # real chunk is not treated as though it could.
363 return int(match.group(1)) <= MAX_CHUNK_INDEX
366# Result-dict key holding a chunk-anchored spelling recorded alongside a
367# citation whose stored link has no anchor. A separate key, never an
368# overwrite of ``link``/``url``: consumers opt in, and a stray write can
369# only add an ignored hint rather than corrupt a citation URL.
370CHUNK_DISPLAY_KEY = "chunk_display_url"
373def preferred_chunk_display(raw: str) -> str | None:
374 """Return the display URL when *raw* is a library route WITH a valid
375 chunk anchor, else ``None``.
377 ``library_display_url`` validates the route and keeps a VALID chunk
378 anchor, dropping any other fragment — a citation without an anchor is
379 still a citation. This is the stricter question a caller asks when it
380 wants to *store* an anchored spelling in preference to one it already
381 has: the anchor has to be present, not merely permitted.
383 (This paragraph said "re-emits the fragment verbatim" until that
384 behaviour was removed — the wording was corrected in the twin function
385 and left stale here, which is the same one-instance-at-a-time slip the
386 functions themselves keep exhibiting.)
388 One helper for both callers on purpose. The collector previously
389 re-derived this rule from a comment saying it matched the renderer's,
390 and the copy drifted twice — once accepting any ``#chunk-`` substring,
391 once validating the route while ignoring the fragment entirely.
392 """
393 display = library_display_url(raw)
394 if display is None:
395 return None
396 _, sep, fragment = display.partition("#")
397 # ``library_display_url`` already drops a fragment that fails this
398 # test, so the ``is_valid_chunk_fragment`` call here is INERT today:
399 # mutating it away kills no test, because nothing can reach it with a
400 # fragment that would fail. It is defence in depth, not a live check —
401 # said plainly so nobody reads a passing test as evidence it fires.
402 #
403 # Kept anyway, deliberately: it makes this function correct
404 # independently of what ``library_display_url`` does, and the two
405 # answer different questions ("what do I render" vs "is this worth
406 # storing"), so a future change to either must not silently make this
407 # one the looser of the pair — which is exactly how the collector's
408 # hand-rolled copy of this rule drifted twice before it was
409 # centralised here. The ``not sep`` half IS load-bearing: it is what
410 # distinguishes "has an anchor" from "has none".
411 if not sep or not is_valid_chunk_fragment(fragment):
412 return None
413 return display
416@lru_cache(maxsize=1024)
417def canonical_url_key(url: str) -> str:
418 """Return a canonical form of ``url`` suitable for deduplication and
419 display in a Sources / citations listing.
421 The canonical form:
422 - lowercases scheme and host (paths stay case-sensitive),
423 - strips userinfo (``user:pass@`` — never leak creds),
424 - strips default ports (80/http, 443/https),
425 - strips fragments,
426 - drops tracking query params (``utm_*``, ``fbclid``, ``gclid``,
427 ``msclkid``, ``yclid``, ``dclid``, ``gad_source``, ``mc_eid``,
428 ``mc_cid``, ``ref_src``, ``igshid``, ``_ga``, ``_gl``),
429 - trims a trailing ``/`` from non-root paths.
431 Click-through behavior is preserved — tracking params carry no
432 content, and mainstream browsers already strip them automatically.
433 Percent-encoding is not normalized; query param order is preserved
434 as-is.
436 Internal library-document routes (``/library/document/<id>``, plus its
437 ``/pdf`` and ``/chunks#chunk-<n>`` views) collapse to a per-document
438 key, because they are views of one source and belong on one
439 bibliography line. That key drops the ``#chunk-<n>`` anchor, so
440 anything that RENDERS a link must display the original URL instead —
441 see :func:`library_display_url`.
443 Falls back to ``url.strip()`` for everything else that is not a
444 recognizable absolute URL (``mailto:``, ``data:``, protocol-relative
445 ``//host/p``, bare filesystem paths, SPA ``/app#/route`` links), since
446 canonicalization would be ambiguous — and merging distinct sources is
447 worse than leaving them separate.
448 """
449 if not url:
450 return ""
451 # Library-document routes — relative, ``/lib/``-abbreviated, or the
452 # absolute ``https://library.document/<id>`` alias the agent emits —
453 # all key to one per-document route, so every view of one document
454 # shares a single bibliography entry.
455 library = _parse_library_citation(url)
456 if library is not None:
457 return library[0]
458 try:
459 parsed = urlsplit(url)
460 except Exception:
461 return url.strip()
462 # Require both a scheme and a netloc; otherwise canonicalization is
463 # ambiguous (mailto:, data:, protocol-relative, etc.).
464 if not parsed.scheme or not parsed.netloc:
465 # Library routes were already handled above. Everything else that
466 # is not a recognizable absolute URL falls back to the raw string:
467 # canonicalization is deliberately NOT generalized to every
468 # root-relative path, because a LangChain retriever sets a
469 # result's url from its ``source`` metadata, which is commonly an
470 # absolute filesystem path. Treating those as routes merges
471 # genuinely distinct sources — ``/docs/C#/a.md`` and
472 # ``/docs/C#/b.md`` would both key to ``/docs/C``, a path that
473 # does not exist — which is worse than the fan-out being fixed.
474 # The same applies to SPA-style ``/app#/route`` links.
475 #
476 # A library route with a control character does NOT reach here as
477 # a library key: ``_parse_library_citation`` refuses it, so it
478 # falls through to this verbatim fallback. That is deliberate.
479 # Truncating the doc-id segment instead would key the crafted URL
480 # as the real ``/library/document/<id>`` it was prefixed with,
481 # merging it into that document's bibliography entry and handing
482 # it the document's title and citation number. The payload cannot
483 # forge a line from here either: every field rendered into the
484 # ``## Sources`` block — this URL included — goes through
485 # ``search_utilities._sanitize_sources_field``, which flattens
486 # control characters to spaces.
487 return url.strip()
489 scheme = parsed.scheme.lower()
491 # Strip userinfo (user:pass@host) from netloc.
492 netloc = parsed.netloc.rsplit("@", 1)[-1]
494 # Split host/port carefully so IPv6 literals survive.
495 if netloc.startswith("["):
496 end = netloc.find("]")
497 host = netloc[: end + 1]
498 rest = netloc[end + 1 :]
499 port = rest[1:] if rest.startswith(":") else ""
500 elif ":" in netloc:
501 host, _, port = netloc.rpartition(":")
502 host = host.lower()
503 else:
504 host, port = netloc.lower(), ""
506 if (scheme == "https" and port == "443") or (
507 scheme == "http" and port == "80"
508 ):
509 port = ""
510 netloc = f"{host}:{port}" if port else host
512 # Filter query params case-insensitively on key; preserve order/values.
513 if parsed.query:
514 pairs = parse_qsl(parsed.query, keep_blank_values=True)
515 kept = [
516 (k, v)
517 for k, v in pairs
518 if not (
519 k.lower() in _TRACKING_PARAMS
520 or any(k.lower().startswith(p) for p in _TRACKING_PREFIXES)
521 )
522 ]
523 query_str = urlencode(kept, doseq=True) if kept else ""
524 else:
525 query_str = ""
527 path = parsed.path
528 if path and path != "/" and path.endswith("/"):
529 path = path.rstrip("/")
531 return urlunsplit((scheme, netloc, path, query_str, ""))
534def is_safe_custom_llm_endpoint(custom_endpoint: Optional[str]) -> bool:
535 """SSRF guard for a user-supplied custom LLM endpoint, applied at the
536 request boundary as fail-fast defense-in-depth.
538 The endpoint is normalized exactly as the OpenAI-compatible provider
539 normalizes it (:func:`normalize_url`), so scheme-less local endpoints
540 such as ``localhost:11434`` or ``192.168.1.10:8000`` are handled the
541 same way the provider handles them, then validated with
542 :func:`validate_url` allowing private IPs / localhost. That accepts
543 local LLM backends (Ollama / LM Studio / vLLM) while still blocking
544 cloud-metadata and link-local targets. An empty / unset endpoint is
545 safe (there is nothing to send to). On rejection a redacted warning
546 is logged (the raw URL may carry credentials).
548 This is not the sole protection: the OpenAI-compatible provider's
549 ``assert_base_url_safe`` re-validates the same URL before the
550 LangChain client is constructed. This guard simply rejects early —
551 before any DB row is written or research thread is spawned — and
552 keeps the endpoint out of the logs.
554 Scope: this validates only the submitted URL. This helper does not validate
555 redirect targets or pin connection-time name resolution.
557 Callers hand this whatever a JSON body contained, so a non-string
558 (other than ``None``) is rejected rather than coerced: it cannot be a
559 usable endpoint, and coercing it would either raise inside ``strip()``
560 or silently treat ``[]`` / ``{}`` as "unset" and persist them.
561 """
562 if custom_endpoint is None:
563 return True
564 if not isinstance(custom_endpoint, str):
565 logger.warning(
566 "SSRF protection: rejected non-string custom_endpoint of type {}",
567 type(custom_endpoint).__name__,
568 )
569 return False
570 endpoint = custom_endpoint.strip()
571 if not endpoint:
572 return True
573 candidate = normalize_url(endpoint)
574 # allow_private_ips=True is deliberate: a self-hosted LLM backend
575 # legitimately lives on 127.0.0.1 or an RFC1918 LAN address, so those must
576 # stay reachable. block_link_local=True is the carve-out inside that --
577 # cloud instance metadata uses link-local ranges, while self-hosted model
578 # servers commonly use localhost or RFC1918 addresses. Blocking the range,
579 # rather than a short literal list, preserves that distinction for IPv4
580 # and IPv6.
581 #
582 # Regression evidence and self-hosted controls:
583 # tests/security/test_llm_endpoint_link_local_hardening.py
584 # - test_link_local_endpoint_is_refused_at_the_http_boundary
585 # - test_self_hosted_endpoint_still_accepted
586 if validate_url(candidate, allow_private_ips=True, block_link_local=True):
587 return True
588 logger.warning(
589 "SSRF protection: rejected custom_endpoint URL: {}",
590 redact_url_for_log(candidate),
591 )
592 return False