Coverage for src/local_deep_research/utilities/url_utils.py: 92%
246 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""URL utility functions for the local deep research application."""
3from functools import lru_cache
4from typing import Optional
5import re
6from urllib.parse import SplitResult, unquote, urlsplit, urlunsplit
8from loguru import logger
10from .chunk_anchor import (
11 LIBRARY_ALIAS_HOST,
12 LIBRARY_ROUTE_PREFIXES,
13 LIBRARY_ROUTE_SUFFIX_ALTERNATION,
14 MAX_CHUNK_INDEX,
15 is_safe_document_id,
16)
17from ..security import redact_url_for_log, validate_url
18from ..security.network_utils import is_private_ip
20# Re-export for backwards compatibility
21__all__ = [
22 "normalize_url",
23 "is_private_ip",
24 "canonical_url_key",
25 "content_web_cache_key",
26 "library_display_url",
27 "is_safe_custom_llm_endpoint",
28]
30# Tracking query parameter keys (matched lowercased).
31_TRACKING_PARAMS = frozenset(
32 {
33 "fbclid",
34 "gclid",
35 "msclkid",
36 "yclid",
37 "dclid",
38 "gad_source",
39 "mc_eid",
40 "mc_cid",
41 "ref_src",
42 "igshid",
43 "_ga",
44 "_gl",
45 }
46)
47# Tracking param name prefixes (matched lowercased).
48_TRACKING_PREFIXES = ("utm_",)
51def normalize_url(raw_url: str) -> str:
52 """
53 Normalize a URL to ensure it has a proper scheme and format.
55 Args:
56 raw_url: The raw URL string to normalize
58 Returns:
59 A properly formatted URL string
61 Examples:
62 >>> normalize_url("localhost:11434")
63 'http://localhost:11434'
64 >>> normalize_url("https://example.com:11434")
65 'https://example.com:11434'
66 >>> normalize_url("http:example.com")
67 'http://example.com'
68 """
69 if not raw_url:
70 raise ValueError("URL cannot be empty")
72 # Clean up the URL
73 raw_url = raw_url.strip()
75 # First check if the URL already has a proper scheme
76 if raw_url.startswith(("http://", "https://")):
77 return raw_url
79 # Handle case where URL is malformed like "http:hostname" (missing //)
80 if raw_url.startswith(("http:", "https:")) and not raw_url.startswith(
81 ("http://", "https://")
82 ):
83 scheme = raw_url.split(":", 1)[0]
84 rest = raw_url.split(":", 1)[1]
85 return f"{scheme}://{rest}"
87 # Handle URLs that start with //
88 if raw_url.startswith("//"):
89 # Remove the // and process
90 raw_url = raw_url[2:]
92 # At this point, we should have hostname:port or just hostname
93 # Determine if this is localhost or an external host
94 hostname = raw_url.split(":")[0].split("/")[0]
96 # Handle IPv6 addresses in brackets
97 if hostname.startswith("[") and "]" in raw_url:
98 # Extract the IPv6 address including brackets
99 hostname = raw_url.split("]")[0] + "]"
101 # Use http for local/private addresses, https for external hosts
102 scheme = "http" if is_private_ip(hostname) else "https"
104 return f"{scheme}://{raw_url}"
107# Internal library-document routes emitted as citation URLs by the RAG
108# search engines (see ``LibraryRAGSearchEngine._get_document_url``). These
109# are the only scheme-less URLs whose shape ``canonical_url_key`` knows
110# well enough to canonicalize; everything else falls back to the raw
111# string. Imported from ``chunk_anchor`` so the producer side (which
112# decides what counts as a library hit) and this consumer side cannot
113# disagree about the prefix set.
114_LIBRARY_ROUTE_PREFIXES = LIBRARY_ROUTE_PREFIXES
117# The absolute alias the agent sometimes emits for a library document; the
118# fetch resolver accepts it (see ``library_resolver._parse_library_reference``)
119# and hands the ORIGINAL string back, which is then registered as a citation
120# link. It must key and display as the relative route it denotes, or one
121# document occupies two bibliography entries.
122_LIBRARY_ALIAS_SCHEME = "https"
123_LIBRARY_ALIAS_HOST = LIBRARY_ALIAS_HOST
124# Mirrors ``library_resolver._LIBRARY_PATH_RE``: a document id plus at most
125# one known view. Anything deeper is not a library route and must not be
126# rewritten into one.
127_LIBRARY_ALIAS_PATH_RE = re.compile(
128 rf"^/(?P<doc_id>[^/?#]+)(?:/(?P<suffix>{LIBRARY_ROUTE_SUFFIX_ALTERNATION}))?/?$"
129)
130# The same shape for the RELATIVE routes, including the ``/lib/`` spelling.
131# Both spellings denote one document (``library_resolver._LIBRARY_PATH_RE``
132# accepts either), so both normalize to the ``/library/`` form: keying them
133# separately fans a document across two bibliography entries, and only
134# ``/library/document/<id>`` is a registered Flask route — a ``/lib/...``
135# link renders dead.
136_LIBRARY_ROUTE_PATH_RE = re.compile(
137 r"^/(?:library|lib)/document/(?P<doc_id>[^/?#]+)"
138 rf"(?:/(?P<suffix>{LIBRARY_ROUTE_SUFFIX_ALTERNATION}))?/?$"
139)
142def _is_library_route(path: str) -> bool:
143 return path.startswith(_LIBRARY_ROUTE_PREFIXES)
146def _has_control_chars(value: str) -> bool:
147 """True if *value* carries a C0/DEL character.
149 These are rendered verbatim into the ``## Sources`` block, where an
150 embedded newline forges extra lines. The library-route branches build
151 their result by string-splitting rather than via ``urlunsplit``, so —
152 unlike the absolute-URL branch — they cannot drop such a character on
153 their own and must reject it explicitly.
154 """
155 return any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in value)
158def _normalize_library_alias(url: str) -> str | None:
159 """Map ``https://library.document/<id>[/pdf|/chunks]`` to its relative route.
161 Returns ``None`` for anything else. The accepted shape is deliberately
162 narrow — a document id followed by at most the ``pdf`` or ``chunks``
163 view, mirroring ``library_resolver._LIBRARY_PATH_RE`` — because the
164 result is keyed AND displayed as an internal route. A permissive
165 rewrite lets a crafted alias merge into a real document's bibliography
166 entry, inherit its title, and replace its link: pasting an unvalidated
167 path after ``/library/document`` turns
168 ``https://library.document/<id>/../../../admin/x#chunk-1`` into a
169 same-origin link to ``/admin/x`` rendered under the real document's
170 name.
172 Two deliberate differences from the resolver, both narrowing or
173 identity-preserving rather than widening:
175 * A query disqualifies (as in the resolver) — it can carry
176 server-meaningful state, unlike a client-side anchor.
177 * Host comparison uses ``hostname``/``port``, so ``LIBRARY.DOCUMENT``,
178 ``:443`` and userinfo forms name the same document rather than
179 fanning out into separate bibliography entries. The resolver rejects
180 those for fetching; they are still the same source for citation
181 purposes.
183 The fragment is carried over — it is the chunk anchor.
184 """
185 # Cheap reject first: this runs once per citation inside
186 # ``format_links_to_markdown``, and almost every real citation is a
187 # plain external URL that can never match. Skipping ``urlsplit`` for
188 # those keeps the bibliography pass off a per-link parse.
189 if "://" not in url or _LIBRARY_ALIAS_HOST not in url.lower():
190 return None
191 # Reject tab/newline/CR before ``urlsplit`` sees them: it deletes them
192 # silently, which would collide ``/a<TAB>b`` with a real ``/ab`` and
193 # rewrite the citation to a DIFFERENT document. The neighbouring
194 # manual splits avoid ``urlsplit`` for exactly this reason.
195 if any(ch in url for ch in "\t\n\r"): 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true
196 return None
197 try:
198 parsed = urlsplit(url)
199 except ValueError:
200 return None
201 if (
202 parsed.scheme != _LIBRARY_ALIAS_SCHEME
203 or parsed.query
204 or (parsed.hostname or "") != _LIBRARY_ALIAS_HOST
205 ):
206 return None
207 try:
208 if parsed.port not in (None, 443):
209 return None
210 except ValueError: # malformed port
211 return None
212 path_match = _LIBRARY_ALIAS_PATH_RE.match(parsed.path)
213 if not path_match:
214 return None
215 doc_id = unquote(path_match.group("doc_id"))
216 # Same predicate ``extract_document_id`` applies, imported rather than
217 # restated: the doc id is decoded before BOTH validating and
218 # re-emitting, so ``/%2D`` and ``/-`` forms of one id give one key.
219 if not is_safe_document_id(doc_id): 219 ↛ 220line 219 didn't jump to line 220 because the condition on line 219 was never true
220 return None
221 suffix = path_match.group("suffix")
222 rel = f"/library/document/{doc_id}"
223 if suffix:
224 rel = f"{rel}/{suffix}"
225 return f"{rel}#{parsed.fragment}" if parsed.fragment else rel
228def _parse_library_citation(raw: str) -> tuple[str, str] | None:
229 """Return ``(dedup_key, display_url)`` for a library citation, else ``None``.
231 One parser for every accepted spelling — the relative
232 ``/library/document/<id>`` route, its ``/lib/`` abbreviation, and the
233 absolute ``https://library.document/<id>`` alias — so the two paths
234 cannot enforce different rules. They used to: only the alias validated
235 its document id, which meant the exact traversal this function's
236 sibling docstring promises to block sailed straight through when the
237 same URL arrived relative. ``/library/document/7/../../../admin/wipe``
238 keyed as document ``7`` and *displayed* as ``/admin/wipe``, so a
239 crafted citation merged into a real document's entry, inherited its
240 title and citation numbers, and replaced its link.
242 * ``dedup_key`` is ``/library/document/<id>`` — the ``/pdf``,
243 ``/chunks`` and ``#chunk-<n>`` views of one document are one source
244 and belong on one bibliography line.
245 * ``display_url`` keeps the view suffix and the ``#chunk-<n>`` anchor,
246 which is what makes a citation scroll to the cited text.
248 Normalisations, all identity-preserving: ``/lib/`` becomes
249 ``/library/``, the document id is percent-decoded (so ``a%2Db`` and
250 ``a-b`` are one key, matching what the resolver looks up), and a query
251 string is dropped (the route ignores it).
252 """
253 if not raw:
254 return None
255 stripped = raw.strip()
256 # Cheap reject before any parsing: the overwhelming majority of
257 # citations are ordinary external URLs.
258 if not (
259 stripped.startswith(_LIBRARY_ROUTE_PREFIXES)
260 or ("://" in stripped and _LIBRARY_ALIAS_HOST in stripped.lower())
261 ):
262 return None
263 # Refuse anything carrying a control character rather than truncating
264 # the doc-id segment at it. Truncating keys the crafted URL as the
265 # real ``/library/document/<id>`` it was prefixed with, merging it
266 # into that document's bibliography entry and lending it the
267 # document's title and citation number. Returning ``None`` keeps the
268 # two sources apart; the payload still cannot forge a Sources line,
269 # because every field rendered into that block goes through
270 # ``search_utilities._sanitize_sources_field``.
271 if _has_control_chars(stripped):
272 return None
273 alias = _normalize_library_alias(stripped)
274 if alias is not None:
275 stripped = alias
276 if not _is_library_route(stripped):
277 return None
278 # Split fragment and query manually rather than via ``urlsplit``,
279 # which silently deletes embedded tab/newline and would collide
280 # ``/a<TAB>b`` with a real ``/ab``.
281 head, sep, fragment = stripped.partition("#")
282 path = head.split("?", 1)[0]
283 path_match = _LIBRARY_ROUTE_PATH_RE.match(path)
284 if not path_match:
285 return None
286 doc_id = unquote(path_match.group("doc_id"))
287 # Same predicate ``extract_document_id`` applies, imported rather than
288 # restated: the id is decoded before BOTH validating and re-emitting,
289 # so ``/%2D`` and ``/-`` forms of one id give one key.
290 if not is_safe_document_id(doc_id):
291 return None
292 key = f"/library/document/{doc_id}"
293 suffix = path_match.group("suffix")
294 display = f"{key}/{suffix}" if suffix else key
295 # Re-attach the fragment ONLY when it is a real chunk anchor. A
296 # library route carries no other meaningful fragment, so anything else
297 # is either a producer bug or an unvalidated string that reached us
298 # from the agent (``fetch_content`` passes its argument through
299 # ``library_resolver`` verbatim). Re-emitting it built a citation URL
300 # whose anchor cannot name a chunk — a dead link that still claims to
301 # cite one — and smuggled arbitrary text into the rendered Sources
302 # block, bypassing the ``MAX_CHUNK_INDEX``/format contract that
303 # ``chunk_anchor`` calls itself the single source of truth for.
304 # Dropping it keeps the citation (the route still resolves to the
305 # document) and loses only an anchor that never pointed anywhere.
306 if sep and fragment and is_valid_chunk_fragment(fragment):
307 display = f"{display}#{fragment}"
308 return key, display
311def library_display_url(raw: str) -> str | None:
312 """Return the display URL when *raw* is a library route, else ``None``.
314 ``canonical_url_key`` collapses library routes to a per-document key,
315 which is right for grouping and wrong for display: the ``#chunk-<n>``
316 anchor is the entire point of a chunk-targeted citation. Renderers
317 group on the key and display this.
319 The returned URL is the *normalized* route (``/library/`` spelling,
320 percent-decoded id, no query), never the caller's raw string — a
321 string that does not parse as a library route returns ``None`` rather
322 than being echoed back into the report.
323 """
324 parsed = _parse_library_citation(raw)
325 return parsed[1] if parsed is not None else None
328# A chunk anchor is ``#chunk-<n>`` with the exact index shape
329# ``build_chunk_anchor_url`` emits. ``\d`` is Unicode-wide, so it is spelled
330# out here: ``chunk-\u0667`` and ``chunk-１２`` would otherwise be accepted
331# while ``chunk_anchor.extract_chunk_index`` — which calls itself the
332# single source of truth for this — rejects them.
333# ``\Z``, not ``$``: Python's ``$`` also matches just before a single
334# trailing newline even without ``re.MULTILINE``, so ``chunk-1\n``
335# passed. Both current callers strip and reject control characters
336# upstream, so nothing reaches here with one today — but this is the
337# shared predicate, and a future caller that skips those gates would
338# inherit a newline-smuggling hole in the one rule both paths trust.
339_CHUNK_FRAGMENT_RE = re.compile(r"^chunk-(0|[1-9][0-9]*)\Z", re.ASCII)
342def is_valid_chunk_fragment(fragment: str) -> bool:
343 """Return ``True`` if *fragment* (the part after ``#``) is a chunk
344 anchor that can actually name a chunk.
346 Shared by :func:`library_display_url` (which drops a fragment failing
347 this test) and :func:`preferred_chunk_display` (which rejects the whole
348 URL), so the render path and the store path cannot disagree about what
349 a valid anchor is.
350 """
351 if not isinstance(fragment, str): 351 ↛ 352line 351 didn't jump to line 352 because the condition on line 351 was never true
352 return False
353 match = _CHUNK_FRAGMENT_RE.match(fragment)
354 if match is None:
355 return False
356 # Bound the digit run BEFORE converting: the pattern admits an
357 # unbounded one, and int() raises above CPython's 4300-digit limit —
358 # out of a pure formatter with no caller catching it, which would take
359 # down the whole Sources block.
360 if len(match.group(1)) > len(str(MAX_CHUNK_INDEX)):
361 return False
362 # Bound it the way the producer does, so a fragment that cannot name a
363 # real chunk is not treated as though it could.
364 return int(match.group(1)) <= MAX_CHUNK_INDEX
367# Result-dict key holding a chunk-anchored spelling recorded alongside a
368# citation whose stored link has no anchor. A separate key, never an
369# overwrite of ``link``/``url``: consumers opt in, and a stray write can
370# only add an ignored hint rather than corrupt a citation URL.
371CHUNK_DISPLAY_KEY = "chunk_display_url"
373# Result-dict key holding the page's own ``<title>``, recorded alongside a
374# citation whose stored title came from a search engine's result listing. A
375# separate key for the same reason as ``CHUNK_DISPLAY_KEY``: never an overwrite.
376FETCHED_TITLE_KEY = "fetched_title"
379def preferred_chunk_display(raw: str) -> str | None:
380 """Return the display URL when *raw* is a library route WITH a valid
381 chunk anchor, else ``None``.
383 ``library_display_url`` validates the route and keeps a VALID chunk
384 anchor, dropping any other fragment — a citation without an anchor is
385 still a citation. This is the stricter question a caller asks when it
386 wants to *store* an anchored spelling in preference to one it already
387 has: the anchor has to be present, not merely permitted.
389 (This paragraph said "re-emits the fragment verbatim" until that
390 behaviour was removed — the wording was corrected in the twin function
391 and left stale here, which is the same one-instance-at-a-time slip the
392 functions themselves keep exhibiting.)
394 One helper for both callers on purpose. The collector previously
395 re-derived this rule from a comment saying it matched the renderer's,
396 and the copy drifted twice — once accepting any ``#chunk-`` substring,
397 once validating the route while ignoring the fragment entirely.
398 """
399 display = library_display_url(raw)
400 if display is None:
401 return None
402 _, sep, fragment = display.partition("#")
403 # ``library_display_url`` already drops a fragment that fails this
404 # test, so the ``is_valid_chunk_fragment`` call here is INERT today:
405 # mutating it away kills no test, because nothing can reach it with a
406 # fragment that would fail. It is defence in depth, not a live check —
407 # said plainly so nobody reads a passing test as evidence it fires.
408 #
409 # Kept anyway, deliberately: it makes this function correct
410 # independently of what ``library_display_url`` does, and the two
411 # answer different questions ("what do I render" vs "is this worth
412 # storing"), so a future change to either must not silently make this
413 # one the looser of the pair — which is exactly how the collector's
414 # hand-rolled copy of this rule drifted twice before it was
415 # centralised here. The ``not sep`` half IS load-bearing: it is what
416 # distinguishes "has an anchor" from "has none".
417 if not sep or not is_valid_chunk_fragment(fragment):
418 return None
419 return display
422def _is_wayback_replay_url(url: str, parsed: SplitResult) -> bool:
423 """Recognize a capture URL whose embedded target must stay verbatim.
425 Capture availability and redirect safety are checked by the archive
426 client. This only keeps citation normalization from changing its target.
427 """
428 if any(character.isspace() or ord(character) < 32 for character in url):
429 return False
430 try:
431 if (
432 parsed.scheme not in {"http", "https"}
433 or parsed.hostname != "web.archive.org"
434 or parsed.port not in {None, 80, 443}
435 or parsed.username is not None
436 or parsed.password is not None
437 ):
438 return False
439 replay = re.fullmatch(
440 r"/web/[0-9]{14}(?:id_|if_)?/(https?://.+)", parsed.path
441 )
442 if replay is None:
443 return False
444 target = urlsplit(replay.group(1))
445 # Match the archive client's absolute, credential-free target
446 # boundary without importing transport code into this utility.
447 if (
448 not target.hostname
449 or target.username is not None
450 or target.password is not None
451 ):
452 return False
453 target.port
454 except ValueError:
455 return False
456 return True
459@lru_cache(maxsize=1024)
460def canonical_url_key(url: str) -> str:
461 """Return a canonical form of ``url`` suitable for deduplication and
462 display in a Sources / citations listing.
464 The canonical form:
465 - lowercases scheme and host (paths stay case-sensitive),
466 - strips userinfo (``user:pass@`` — never leak creds),
467 - strips default ports (80/http, 443/https),
468 - strips fragments,
469 - drops tracking query params (``utm_*``, ``fbclid``, ``gclid``,
470 ``msclkid``, ``yclid``, ``dclid``, ``gad_source``, ``mc_eid``,
471 ``mc_cid``, ``ref_src``, ``igshid``, ``_ga``, ``_gl``),
472 - trims a trailing ``/`` from non-root paths.
474 Click-through behavior is preserved — tracking params carry no
475 content, and mainstream browsers already strip them automatically.
476 Percent-encoding is not normalized: the raw query spellings
477 ``?q=a+b`` vs ``?q=a%20b`` and ``?a=1;b=2`` vs ``?a=1%3Bb=2`` hash
478 differently (a server may treat ``+``/``;`` literally, so merging
479 them would risk linking a resource to the wrong document). Query
480 param order is preserved as-is here; callers that need
481 order-insensitivity (e.g. download-tracker hashing) sort afterwards.
483 Timestamped Wayback replay URLs retain their complete spelling,
484 including the capture modifier and embedded target's path, query and
485 fragment. Removing a trailing slash or a tracking parameter there
486 changes the captured target rather than merely cleaning a live link.
488 Internal library-document routes (``/library/document/<id>``, plus its
489 ``/pdf`` and ``/chunks#chunk-<n>`` views) collapse to a per-document
490 key, because they are views of one source and belong on one
491 bibliography line. That key drops the ``#chunk-<n>`` anchor, so
492 anything that RENDERS a link must display the original URL instead —
493 see :func:`library_display_url`.
495 Falls back to ``url.strip()`` for everything else that is not a
496 recognizable absolute URL (``mailto:``, ``data:``, protocol-relative
497 ``//host/p``, bare filesystem paths, SPA ``/app#/route`` links), since
498 canonicalization would be ambiguous — and merging distinct sources is
499 worse than leaving them separate.
500 """
501 if not url:
502 return ""
503 # Library-document routes — relative, ``/lib/``-abbreviated, or the
504 # absolute ``https://library.document/<id>`` alias the agent emits —
505 # all key to one per-document route, so every view of one document
506 # shares a single bibliography entry.
507 library = _parse_library_citation(url)
508 if library is not None:
509 return library[0]
510 try:
511 parsed = urlsplit(url)
512 except Exception:
513 return url.strip()
514 # Require both a scheme and a netloc; otherwise canonicalization is
515 # ambiguous (mailto:, data:, protocol-relative, etc.).
516 if not parsed.scheme or not parsed.netloc:
517 # Library routes were already handled above. Everything else that
518 # is not a recognizable absolute URL falls back to the raw string:
519 # canonicalization is deliberately NOT generalized to every
520 # root-relative path, because a LangChain retriever sets a
521 # result's url from its ``source`` metadata, which is commonly an
522 # absolute filesystem path. Treating those as routes merges
523 # genuinely distinct sources — ``/docs/C#/a.md`` and
524 # ``/docs/C#/b.md`` would both key to ``/docs/C``, a path that
525 # does not exist — which is worse than the fan-out being fixed.
526 # The same applies to SPA-style ``/app#/route`` links.
527 #
528 # A library route with a control character does NOT reach here as
529 # a library key: ``_parse_library_citation`` refuses it, so it
530 # falls through to this verbatim fallback. That is deliberate.
531 # Truncating the doc-id segment instead would key the crafted URL
532 # as the real ``/library/document/<id>`` it was prefixed with,
533 # merging it into that document's bibliography entry and handing
534 # it the document's title and citation number. The payload cannot
535 # forge a line from here either: every field rendered into the
536 # ``## Sources`` block — this URL included — goes through
537 # ``search_utilities._sanitize_sources_field``, which flattens
538 # control characters to spaces.
539 return url.strip()
541 if _is_wayback_replay_url(url, parsed):
542 return url
544 scheme = parsed.scheme.lower()
546 # Strip userinfo (user:pass@host) from netloc.
547 netloc = parsed.netloc.rsplit("@", 1)[-1]
549 # Split host/port carefully so IPv6 literals survive.
550 if netloc.startswith("["):
551 end = netloc.find("]")
552 raw_host = netloc[: end + 1]
553 # Lowercase IPv6 address portion while preserving case-sensitive zone ID (RFC 6874)
554 if "%" in raw_host:
555 addr, zone = raw_host.split("%", 1)
556 host = f"{addr.lower()}%{zone}"
557 else:
558 host = raw_host.lower()
559 rest = netloc[end + 1 :]
560 port = rest[1:] if rest.startswith(":") else ""
561 elif ":" in netloc:
562 host, _, port = netloc.rpartition(":")
563 host = host.lower()
564 else:
565 host, port = netloc.lower(), ""
567 if (scheme == "https" and port == "443") or (
568 scheme == "http" and port == "80"
569 ):
570 port = ""
571 netloc = f"{host}:{port}" if port else host
573 # Filter query params case-insensitively on key; preserve raw
574 # spellings and order. Operates on the raw ``&``-split pairs (no
575 # parse_qsl/urlencode round-trip): decoding would merge spellings the
576 # tracker must keep distinct (``+`` vs ``%20``, ``;`` vs ``%3B`` —
577 # issue #6758 B2 review). Tracking detection decodes the key with
578 # ``unquote`` (``%`` only, ``+`` left intact) so an encoded tracking
579 # key is still stripped, but the kept pairs are re-emitted verbatim.
580 if parsed.query:
581 kept_raw: list[str] = []
582 for raw_pair in parsed.query.split("&"):
583 if raw_pair == "": 583 ↛ 584line 583 didn't jump to line 584 because the condition on line 583 was never true
584 continue
585 raw_key = raw_pair.split("=", 1)[0]
586 try:
587 decoded_key = unquote(raw_key).lower()
588 except Exception:
589 decoded_key = raw_key.lower()
590 if decoded_key in _TRACKING_PARAMS or any(
591 decoded_key.startswith(p) for p in _TRACKING_PREFIXES
592 ):
593 continue
594 kept_raw.append(raw_pair)
595 query_str = "&".join(kept_raw)
596 else:
597 query_str = ""
599 path = parsed.path
600 if path and path != "/" and path.endswith("/"):
601 path = path.rstrip("/")
603 return urlunsplit((scheme, netloc, path, query_str, ""))
606@lru_cache(maxsize=1024)
607def content_web_cache_key(url: str) -> str:
608 """Return a cache key for *web* content fetches (NOT bibliography dedup).
610 Same normalisations as :func:`canonical_url_key` — lowercased
611 scheme/host, stripped default ports, dropped tracking query params —
612 but with deliberate differences, because bibliographic equivalence
613 does not establish identical fetched bytes:
615 * the path is preserved verbatim, including a non-root trailing
616 ``/``: ``/article`` and ``/article/`` are different HTTP resources
617 and routinely serve different bodies;
618 * URL userinfo (``user:pass@``) is preserved verbatim (host still
619 lowercased): ``https://alice@host/p`` and ``https://bob@host/p``
620 authenticate differently and must never share an entry;
621 * the fragment is preserved verbatim: client-routed SPAs (hash
622 routing) render different views per fragment under JS rendering
623 (Playwright navigates with the fragment intact).
625 Query handling otherwise matches the bibliography key: only tracking
626 params (``utm_*``, ``fbclid``, ...) collapse; every other param,
627 its order and its values are preserved, so ``?a=1`` and ``?a=2``
628 stay distinct. Dropping tracking parameters accepts the trade-off of
629 potential over-merge on rare sites that vary content per tracking param
630 (e.g., campaign- or affiliate-specific landing pages). Raw spellings
631 (``?q=a+b`` vs ``?q=a%20b``) stay
632 distinct, matching :func:`canonical_url_key` (#6758 B2). Anything
633 without a scheme+netloc falls back to the stripped raw string (same as
634 :func:`canonical_url_key`).
635 Library routes are NOT handled here — callers resolve those via
636 ``library_display_url`` first. A relative ``/library/...`` URL
637 therefore falls through to the raw fallback, never to a web key.
638 """
640 if not url: 640 ↛ 641line 640 didn't jump to line 641 because the condition on line 640 was never true
641 return ""
642 try:
643 parsed = urlsplit(url)
644 except Exception:
645 return url.strip()
646 if not parsed.scheme or not parsed.netloc: 646 ↛ 647line 646 didn't jump to line 647 because the condition on line 646 was never true
647 return url.strip()
649 scheme = parsed.scheme.lower()
651 # Preserve userinfo verbatim (credentials are case-sensitive and
652 # select a different origin view); lowercase only the host part.
653 raw_netloc = parsed.netloc
654 if "@" in raw_netloc:
655 userinfo, _, hostport = raw_netloc.rpartition("@")
656 else:
657 userinfo, hostport = "", raw_netloc
659 # Split host/port carefully so IPv6 literals survive.
660 if hostport.startswith("["):
661 end = hostport.find("]")
662 if end == -1: 662 ↛ 663line 662 didn't jump to line 663 because the condition on line 662 was never true
663 return url.strip()
664 host = hostport[: end + 1].lower()
665 rest = hostport[end + 1 :]
666 port = rest[1:] if rest.startswith(":") else ""
667 if rest and not rest.startswith(":"): 667 ↛ 668line 667 didn't jump to line 668 because the condition on line 667 was never true
668 return url.strip()
669 elif ":" in hostport:
670 host, _, port = hostport.rpartition(":")
671 host = host.lower()
672 else:
673 host, port = hostport.lower(), ""
675 if (scheme == "https" and port == "443") or (
676 scheme == "http" and port == "80"
677 ):
678 port = ""
679 hostport_norm = f"{host}:{port}" if port else host
680 netloc = f"{userinfo}@{hostport_norm}" if userinfo else hostport_norm
682 # Filter query params case-insensitively on key; preserve raw
683 # spellings and order (same raw-preserving filter as
684 # :func:`canonical_url_key` — no parse_qsl/urlencode round-trip).
685 if parsed.query:
686 kept_raw: list[str] = []
687 for raw_pair in parsed.query.split("&"):
688 if raw_pair == "": 688 ↛ 689line 688 didn't jump to line 689 because the condition on line 688 was never true
689 continue
690 raw_key = raw_pair.split("=", 1)[0]
691 try:
692 decoded_key = unquote(raw_key).lower()
693 except Exception:
694 decoded_key = raw_key.lower()
695 if decoded_key in _TRACKING_PARAMS or any(
696 decoded_key.startswith(p) for p in _TRACKING_PREFIXES
697 ):
698 continue
699 kept_raw.append(raw_pair)
700 query_str = "&".join(kept_raw)
701 else:
702 query_str = ""
704 # Path preserved verbatim: case AND trailing slash both select
705 # distinct HTTP resources. Fragment preserved: client-routed SPAs
706 # (hash routing) render different content per fragment under JS
707 # rendering (playwright_html navigates with fragment intact).
708 path = parsed.path
709 fragment = parsed.fragment
711 return urlunsplit((scheme, netloc, path, query_str, fragment))
714def is_safe_custom_llm_endpoint(custom_endpoint: Optional[str]) -> bool:
715 """SSRF guard for a user-supplied custom LLM endpoint, applied at the
716 request boundary as fail-fast defense-in-depth.
718 The endpoint is normalized exactly as the OpenAI-compatible provider
719 normalizes it (:func:`normalize_url`), so scheme-less local endpoints
720 such as ``localhost:11434`` or ``192.168.1.10:8000`` are handled the
721 same way the provider handles them, then validated with
722 :func:`validate_url` allowing private IPs / localhost. That accepts
723 local LLM backends (Ollama / LM Studio / vLLM) while still blocking
724 cloud-metadata and link-local targets. An empty / unset endpoint is
725 safe (there is nothing to send to). On rejection a redacted warning
726 is logged (the raw URL may carry credentials).
728 This is not the sole protection: the OpenAI-compatible provider's
729 ``assert_base_url_safe`` re-validates the same URL before the
730 LangChain client is constructed. This guard simply rejects early —
731 before any DB row is written or research thread is spawned — and
732 keeps the endpoint out of the logs.
734 Scope: this validates only the submitted URL. This helper does not validate
735 redirect targets or pin connection-time name resolution.
737 Callers hand this whatever a JSON body contained, so a non-string
738 (other than ``None``) is rejected rather than coerced: it cannot be a
739 usable endpoint, and coercing it would either raise inside ``strip()``
740 or silently treat ``[]`` / ``{}`` as "unset" and persist them.
741 """
742 if custom_endpoint is None:
743 return True
744 if not isinstance(custom_endpoint, str):
745 logger.warning(
746 "SSRF protection: rejected non-string custom_endpoint of type {}",
747 type(custom_endpoint).__name__,
748 )
749 return False
750 endpoint = custom_endpoint.strip()
751 if not endpoint:
752 return True
753 candidate = normalize_url(endpoint)
754 # allow_private_ips=True is deliberate: a self-hosted LLM backend
755 # legitimately lives on 127.0.0.1 or an RFC1918 LAN address, so those must
756 # stay reachable. block_link_local=True is the carve-out inside that --
757 # cloud instance metadata uses link-local ranges, while self-hosted model
758 # servers commonly use localhost or RFC1918 addresses. Blocking the range,
759 # rather than a short literal list, preserves that distinction for IPv4
760 # and IPv6.
761 #
762 # Regression evidence and self-hosted controls:
763 # tests/security/test_llm_endpoint_link_local_hardening.py
764 # - test_link_local_endpoint_is_refused_at_the_http_boundary
765 # - test_self_hosted_endpoint_still_accepted
766 if validate_url(candidate, allow_private_ips=True, block_link_local=True):
767 return True
768 logger.warning(
769 "SSRF protection: rejected custom_endpoint URL: {}",
770 redact_url_for_log(candidate),
771 )
772 return False