Coverage for src/local_deep_research/utilities/chunk_anchor.py: 98%
102 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""Helpers for building chunk-targeted library/RAG citation anchors.
3Centralizes the validation and URL-fragment construction that RAG search
4engines and the LangGraph result collector all share. Every producer
5must route chunk metadata through :func:`extract_chunk_index` before
6interpolating it into a ``#chunk-...`` URL fragment — a UUID or boolean
7``chunk_id`` would otherwise slip into ``document_chunks.html`` as a
8fragment that points at a chunk that does not exist. This module is the
9single source of truth so producers and the collector agree on what a
10valid chunk index looks like.
11"""
13from __future__ import annotations
15import re
16from typing import Any, Mapping, Optional
19# The document views a library route may address. Shared by the two
20# route-matching regexes — ``library_resolver._LIBRARY_PATH_RE`` and
21# ``url_utils._LIBRARY_ROUTE_PATH_RE`` — so adding a view teaches both at
22# once instead of relying on one to be hand-mirrored from the other.
23#
24# It does NOT unify every route predicate in the codebase:
25# ``url_utils._is_library_route`` is a prefix test that answers a
26# different question (is this string shaped like a library route at all)
27# and is deliberately looser. Do not read this constant as making all
28# three agree.
29LIBRARY_ROUTE_SUFFIXES = ("pdf", "chunks")
30# Escaped: these are interpolated into regexes, so a future view containing
31# a metacharacter (``v1.2``, ``raw+text``) would otherwise silently widen
32# what counts as a library route.
33LIBRARY_ROUTE_SUFFIX_ALTERNATION = "|".join(
34 re.escape(suffix) for suffix in LIBRARY_ROUTE_SUFFIXES
35)
37# Root-relative routes the library / collection RAG engines emit as citation
38# URLs, plus the absolute alias the agent sometimes types instead. Shared
39# with ``utilities.url_utils`` (which imports from here — this module stays
40# dependency-free so the import can only go one way).
41LIBRARY_ROUTE_PREFIXES = ("/library/document/", "/lib/document/")
42LIBRARY_ALIAS_HOST = "library.document"
43_LIBRARY_ALIAS_SCHEME_PREFIX = "https://"
45# Chunk indices are small ordinals within one document. The bound is far
46# above any realistic chunk count and exists so an absurd value (a
47# timestamp, a float that rounded to 10**30, a 64-bit id mistaken for an
48# index) cannot be interpolated into a URL fragment.
49MAX_CHUNK_INDEX = 1_000_000
51# Document ids are UUIDs / hex hashes / slugs. ASCII only: ``str.isalnum()``
52# accepts every Unicode letter and digit, so a fullwidth "\uff17" would pass
53# and then key separately from its percent-encoded form.
54_SAFE_ID_RE = re.compile(r"^[0-9A-Za-z_-]+$")
57def extract_chunk_index(metadata: Mapping[str, Any] | None) -> Optional[int]:
58 """Return a validated non-negative chunk index from a result's metadata,
59 or ``None`` if the metadata does not carry a usable value.
61 Reads (in order):
63 * ``metadata["chunk_index"]`` — preferred, always an ``int`` in the
64 production schema.
65 * ``metadata["chunk_id"]`` — legacy field that may carry either an
66 ``int`` or an int-as-string. UUIDs and other non-int-like values
67 are rejected.
69 Accepts:
71 * ``int`` (excluding ``bool``) — used as-is.
72 * ``str`` composed solely of ASCII digits.
73 * ``float`` whose value is a whole non-negative integer (catches
74 e.g. ``0.0`` from numpy conversion).
76 Rejects (returns ``None``):
78 * ``bool`` (avoids ``True`` being printed as ``#chunk-True``).
79 * Negative integers — zero is allowed because chunks are 0-indexed.
80 * Values above :data:`MAX_CHUNK_INDEX`. No document has a million
81 chunks, so a value that large is a timestamp, a 64-bit id, or the
82 silent binary rounding of a float literal such as ``1e30`` — none of
83 which addresses a real anchor.
84 * Strings that are not pure ASCII digits: ``"550e8400-..."`` UUIDs,
85 signed forms (``"+5"``/``"-1"``), surrounding whitespace (``" 5 "``),
86 and non-ASCII digit characters that ``str.isdigit()`` accepts but
87 that no anchor id can contain (Arabic-Indic ``"\u0667"``,
88 mathematical ``"\U0001d7dd"``, ...).
89 * Floats with a fractional part.
90 * Anything else (``None``, dicts, lists, ...).
91 """
92 if not isinstance(metadata, Mapping):
93 return None
95 chunk_idx_raw: Any = None
96 if metadata.get("chunk_index") is not None:
97 chunk_idx_raw = metadata["chunk_index"]
98 elif metadata.get("chunk_id") is not None:
99 chunk_idx_raw = metadata["chunk_id"]
100 else:
101 return None
103 try:
104 if isinstance(chunk_idx_raw, bool):
105 # bool is a subclass of int but True/False are never chunk ids.
106 return None
107 if isinstance(chunk_idx_raw, int):
108 value = chunk_idx_raw
109 elif isinstance(chunk_idx_raw, str):
110 # Reject surrounding whitespace, signs, and every non-ASCII
111 # digit: the value is interpolated verbatim into an anchor, and
112 # ``str.isdigit()`` alone is Unicode-wide (it accepts "\u0667"
113 # and "\U0001d7dd", which ``int()`` then happily converts to 7
114 # and 5). Deliberately no ``.strip()`` — the docstring has
115 # always promised whitespace is rejected.
116 if not chunk_idx_raw.isascii() or not chunk_idx_raw.isdigit():
117 return None
118 value = int(chunk_idx_raw)
119 elif isinstance(chunk_idx_raw, float):
120 if not chunk_idx_raw.is_integer():
121 return None
122 value = int(chunk_idx_raw)
123 else:
124 return None
125 except (ValueError, AttributeError):
126 return None
128 # Negative chunk ids never match a real chunk — reject. Zero is
129 # allowed: the document_chunks.html template renders ``chunk.index``
130 # directly as the HTML anchor id, so a 0-indexed chunk anchors at
131 # ``#chunk-0``.
132 if value < 0 or value > MAX_CHUNK_INDEX:
133 return None
134 return value
137def extract_document_id(
138 metadata: Mapping[str, Any] | None, *top_level: Any
139) -> Optional[str]:
140 """Return a sanitised library document id from a result's metadata or
141 top-level fields, or ``None`` if no usable id is present.
143 Reads (in order):
145 * ``metadata["doc_id"]`` — legacy key.
146 * ``metadata["source_id"]`` — used by the current LibraryRAGSearchEngine.
147 * ``metadata["document_id"]`` — alternate key.
148 * ``top_level[0]``'s ``source_id`` / ``document_id`` — for results that
149 promote the id to the top-level dict instead of metadata.
151 Sanitisation:
153 * Non-strings are coerced: ``int`` → ``str``.
154 * Strings are stripped; non-empty values containing only
155 alphanumerics, dashes, and underscores pass. Anything else
156 (whitespace, path traversal, control chars, ``/``) is rejected
157 because it would be interpolated into a URL path.
158 """
159 candidates: list[Any] = []
160 if isinstance(metadata, Mapping):
161 candidates.append(metadata.get("doc_id"))
162 candidates.append(metadata.get("source_id"))
163 candidates.append(metadata.get("document_id"))
164 for obj in top_level:
165 if isinstance(obj, Mapping):
166 candidates.append(obj.get("doc_id"))
167 candidates.append(obj.get("source_id"))
168 candidates.append(obj.get("document_id"))
170 for raw in candidates:
171 if raw is None:
172 continue
173 if isinstance(raw, int) and not isinstance(raw, bool):
174 return str(raw)
175 if isinstance(raw, str):
176 stripped = raw.strip()
177 if not stripped:
178 continue
179 if not is_safe_document_id(stripped):
180 continue
181 return stripped
182 return None
185def is_safe_document_id(value: str) -> bool:
186 """Return ``True`` if *value* is safe to interpolate into a URL path.
188 **ASCII** alphanumerics, dashes and underscores only — which excludes
189 ``/``, ``?``, ``#``, ``%``, whitespace, control characters and ``..``
190 traversal. ASCII matters: ``str.isalnum()`` is true for every Unicode
191 letter and digit, so a fullwidth ``"\uff17"`` would pass, be emitted
192 unencoded into a URL path, and then key separately from its
193 ``%EF%BC%97`` form — one document, two bibliography entries.
195 Shared by :func:`extract_document_id`, :func:`build_chunk_anchor_url`
196 and ``url_utils``'s library-route parser so none of them can drift
197 apart.
198 """
199 return isinstance(value, str) and _SAFE_ID_RE.match(value) is not None
202def is_library_chunk_result(result: Mapping[str, Any] | None) -> bool:
203 """Return ``True`` if *result* is a library/RAG hit that should receive
204 a chunk anchor.
206 Heuristic: the result's ``source`` / ``source_type`` field is
207 ``"library"`` or its link *is* a library-document route (see
208 :func:`is_library_document_link`). Mirrors the producer-side checks in
209 :class:`langgraph_agent_strategy.SearchResultsCollector` so the two
210 agree on the eligibility rule.
211 """
212 if not isinstance(result, Mapping):
213 return False
214 if result.get("source") == "library":
215 return True
216 if result.get("source_type") == "library":
217 return True
218 return is_library_document_link(result.get("link") or result.get("url"))
221def _alias_host(stripped: str) -> str | None:
222 """Return the lowercased host of an ``https://<host>/...`` URL, or ``None``.
224 Userinfo and port are removed so ``https://u@library.document:443/x``
225 and ``https://library.document/x`` give the same host. A trailing
226 ``/`` is required, so a bare authority with no path is not a document
227 link — matching what the literal-prefix test accepted before.
228 """
229 if not stripped.lower().startswith(_LIBRARY_ALIAS_SCHEME_PREFIX):
230 return None
231 rest = stripped[len(_LIBRARY_ALIAS_SCHEME_PREFIX) :]
232 authority, sep, _ = rest.partition("/")
233 if not sep:
234 return None
235 # Userinfo may itself contain ``@``; the host is after the LAST one.
236 authority = authority.rpartition("@")[2]
237 if authority.startswith("["): 237 ↛ 239line 237 didn't jump to line 239 because the condition on line 237 was never true
238 # IPv6 literal: the port, if any, follows the closing bracket.
239 host = authority.partition("]")[0] + "]"
240 else:
241 host = authority.partition(":")[0]
242 return host.lower()
245def is_library_document_link(link: Any) -> bool:
246 """Return ``True`` if *link* addresses a local library document.
248 Anchored at the start of the URL on purpose. A substring test
249 (``"/library/document/" in link``) also matches
250 ``https://evil.example/library/document/7/chunks``, and the caller
251 reacts by REPLACING the link with a local route — silently relabelling
252 an external result as a document in the user's own library.
253 """
254 if not isinstance(link, str):
255 return False
256 stripped = link.strip()
257 if stripped.startswith(LIBRARY_ROUTE_PREFIXES):
258 return True
259 # The absolute alias the agent sometimes emits. Matched on the HOST,
260 # not on a literal ``https://library.document/`` prefix: the prefix
261 # form misses the ``:443`` and userinfo spellings that
262 # ``url_utils._normalize_library_alias`` deliberately accepts as the
263 # same document, so a caller asking "is this a library citation" got
264 # False for a string the renderer would happily normalise. Callers
265 # that strip an unusable fragment then skipped those spellings
266 # entirely, and the raw value reached the DB and the MCP payload.
267 #
268 # Parsed by hand rather than with ``urlsplit``, which silently DELETES
269 # embedded tab/newline/CR — the very characters that make a crafted
270 # alias worth catching here, and which would otherwise let
271 # ``library.doc\tument`` answer for ``library.document``.
272 return _alias_host(stripped) == LIBRARY_ALIAS_HOST
275def build_chunk_anchor_url(
276 link: str,
277 doc_id: Optional[str],
278 chunk_index: Optional[int],
279) -> Optional[str]:
280 """Build a ``/library/document/<doc_id>/chunks#chunk-<n>`` URL when
281 *doc_id* and *chunk_index* are both valid, otherwise return ``None``.
283 Returns ``None`` (rather than mutating *link*) when:
285 * the chunk index failed validation (UUID, bool, negative, string,
286 float with a fractional part, or any other non-int-like value), or
287 * the document id is missing, or is not composed solely of
288 alphanumerics, dashes and underscores (see
289 :func:`is_safe_document_id`).
291 Callers should leave the original *link* unchanged on ``None`` —
292 appending ``#chunk-...`` to whatever route was already in the link
293 would point the anchor at an unrelated route when the doc id failed
294 sanitisation.
295 """
296 # Re-validate ``chunk_index`` defensively. Most callers should have
297 # already passed it through :func:`extract_chunk_index`, but a
298 # future caller that passes a raw value (e.g. a producer that
299 # forgot to validate) must not be able to inject a malformed
300 # fragment. ``int`` excludes ``bool`` (which is technically an
301 # ``int`` subclass) and ``None``.
302 # Enforce the doc-id contract here rather than trusting the caller.
303 # Every in-tree caller runs ``extract_document_id`` first, but this
304 # function BUILDS the URL, so an unvalidated id would be interpolated
305 # straight into the path — a ``../..`` id yields a traversal route,
306 # and an id containing ``#`` yields two fragments. The
307 # docstring above has always promised a rejected id returns ``None``;
308 # this makes that true.
309 # Accept exactly what ``extract_document_id`` accepts — ``int``
310 # (excluding ``bool``) or ``str`` — so the two genuinely cannot drift,
311 # and build the URL from the STRIPPED value that was validated rather
312 # than from the raw argument, or leading/trailing control characters
313 # survive into the returned URL.
314 if doc_id is None or isinstance(doc_id, bool):
315 return None
316 if not isinstance(doc_id, (int, str)):
317 return None
318 safe_doc_id = str(doc_id).strip()
319 if not is_safe_document_id(safe_doc_id):
320 return None
321 if (
322 chunk_index is None
323 or isinstance(chunk_index, bool)
324 or not isinstance(chunk_index, int)
325 or chunk_index < 0
326 or chunk_index > MAX_CHUNK_INDEX
327 ):
328 return None
329 return f"/library/document/{safe_doc_id}/chunks#chunk-{chunk_index}"