Coverage for src/local_deep_research/advanced_search_system/tools/fetch/library_resolver.py: 95%
122 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""Resolve ``fetch_content`` URLs that aren't actually network URLs.
3The LangGraph agent's ``fetch_content`` tool receives two URL-shaped
4strings that the egress policy correctly rejects as
5``unsupported_scheme`` (and that the original ``ContentFetcher`` would
6not know how to fetch even without a policy gate):
81. **Local library document references** like
9 ``/library/document/<uuid>``, ``/library/document/<uuid>/chunks#chunk-3``,
10 ``/lib/document/<uuid>``, ``https://library.document/<uuid>``, or
11 ``[<uuid>]``.
12 The library RAG search engine and the collection search engine emit
13 these as the citation URL of a search hit (see
14 ``web_search_engines.engines.search_engine_library.LibraryRAGSearchEngine.search``
15 and ``...search_engine_collection.CollectionSearchEngine._get_document_url``).
16 The URL points at the user's own library — a local DB read, not a
17 network fetch — and resolving it via ``DocumentChunk.chunk_text`` (when
18 a chunk anchor is provided) or ``Document.text_content`` is what
19 the human user would see if they clicked the link in the UI.
212. **Bare citation markers** like ``[1062]``, ``[1084]``. The agent
22 sometimes pastes a citation marker it saw in the search-results block
23 back into the tool instead of the actual URL. ``SearchResultsCollector``
24 already maps URL → index; the inverse ``find_by_index`` lets the tool
25 resolve ``[N]`` back to its source.
27Both shapes are intercepted here BEFORE the egress policy gate, so the
28agent gets the real page content (or a helpful error) instead of a
29generic "blocked by egress policy" denial that wastes a fetch slot
30(A3 in ``research_f3045c5b_issue_analysis.md``).
32Both the path and bracket forms bind the document id to the canonical
3332/64-hex-or-UUID pattern, so a guessable filename (``report.pdf``) is
34rejected at the regex instead of relying on a downstream DB miss.
35Resolution always runs inside ``get_user_db_session(username)``, so a
36parsed reference can still only return a document the caller already owns.
37"""
39from __future__ import annotations
41import re
42import uuid
43from dataclasses import dataclass
44from typing import Any, Callable, Optional
45from urllib.parse import unquote, urlsplit
47from loguru import logger
49from ....utilities.chunk_anchor import (
50 LIBRARY_ROUTE_SUFFIX_ALTERNATION,
51 is_safe_document_id,
52)
53from ....utilities.url_utils import is_valid_chunk_fragment
55# Canonical document IDs are 32/64-hex blobs or a UUID. The path form is
56# bound to this same pattern as the bracket form (below) so a guessable
57# filename like ``/library/document/report.pdf`` is rejected at the regex
58# rather than reaching the DB only to miss.
59_DOCUMENT_ID_PATTERN = (
60 r"(?:"
61 r"[0-9a-fA-F]{32}|"
62 r"[0-9a-fA-F]{64}|"
63 r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
64 r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
65 r")"
66)
67# Match canonical paths and the ``/lib/document/...`` abbreviation observed
68# in agent tool calls. The same suffix is used by download_service.
69#
70# ``chunks`` is accepted alongside ``pdf`` because the library/collection
71# RAG engines cite chunk-targeted URLs (``/library/document/<id>/chunks``
72# plus a ``#chunk-<n>`` fragment) since #5381.
73_LIBRARY_PATH_RE = re.compile(
74 rf"^/(?:library|lib)/document/(?P<doc_id>{_DOCUMENT_ID_PATTERN})"
75 rf"(?:/(?P<suffix>{LIBRARY_ROUTE_SUFFIX_ALTERNATION}))?/?$"
76)
77_BRACKETED_DOCUMENT_RE = re.compile(
78 rf"^\[(?P<doc_id>{_DOCUMENT_ID_PATTERN})\]$"
79)
81# Match ``[N]`` — a 1-based citation marker with no surrounding chars.
82# The agent emits this verbatim when it confuses the citation marker for
83# a URL; we resolve it via SearchResultsCollector.find_by_index.
84_CITATION_REF_RE = re.compile(r"^\[(\d+)\]$")
87@dataclass(frozen=True)
88class _LibraryReference:
89 value: str
90 suffix: Optional[str] = None
91 #: True when the reference arrived as the bare ``[<uuid>]`` alias
92 #: rather than as a route. That spelling is not a URL at all, so
93 #: registering it verbatim as the citation link gives the document a
94 #: THIRD bibliography entry alongside its ``/library/document/<id>``
95 #: and ``https://library.document/<id>`` citations — and renders a
96 #: dead link. Callers re-emit the canonical route instead.
97 bracketed: bool = False
98 chunk_index: Optional[int] = None
101def _decode_segment(value: str) -> str | None:
102 """Percent-decode one path segment, refusing separators and controls.
104 Now that ``_LIBRARY_PATH_RE``'s doc_id group is bound to hex/UUID, the only
105 live call site can never hand this a ``%``, so ``unquote`` is a no-op there
106 and none of the checks below fire. It is kept deliberately rather than
107 inlined away: it is the guard that makes loosening that group safe, and
108 deleting it would silently remove the safety net for whoever does. Anyone
109 widening the doc_id charset should confirm this is reachable again.
110 """
111 decoded = unquote(value)
112 if ( 112 ↛ 117line 112 didn't jump to line 117 because the condition on line 112 was never true
113 not decoded
114 or any(char in decoded for char in ("/", "\\", "?", "#"))
115 or any(ord(char) < 32 or ord(char) == 127 for char in decoded)
116 ):
117 return None
118 return decoded
121def _parse_library_reference(url: str) -> _LibraryReference | None:
122 if not isinstance(url, str):
123 return None
125 candidate = url.strip()
126 if not candidate:
127 return None
129 # Extract any validated ``#chunk-<n>`` fragment before splitting the path.
130 # Chunk citations carry ``#chunk-<n>``, which addresses a specific chunk.
131 # We record the index and strip the fragment before the ``://`` test so that
132 # (a) the ``https://library.document/<id>/chunks#chunk-3`` alias is not
133 # rejected by the absolute branch's no-fragment rule, and (b) a
134 # relative path whose fragment happens to contain ``://``
135 # (``/library/document/<id>#http://x``) is not misrouted into it.
136 chunk_index: Optional[int] = None
137 if "#" in candidate:
138 head, fragment = candidate.split("#", 1)
139 candidate = head
140 if is_valid_chunk_fragment(fragment):
141 chunk_index = int(fragment[len("chunk-") :])
142 if not candidate:
143 return None
145 if "://" in candidate:
146 parsed = urlsplit(candidate)
147 # No ``parsed.fragment`` check: the fragment was stripped above, so
148 # it can never reach here. A query still disqualifies — it can carry
149 # server-meaningful state, unlike a client-side anchor.
150 if (
151 parsed.scheme != "https"
152 or parsed.netloc != "library.document"
153 or parsed.query
154 ):
155 return None
156 candidate = f"/library/document{parsed.path}"
158 path_match = _LIBRARY_PATH_RE.match(candidate)
159 if path_match:
160 doc_id = _decode_segment(path_match.group("doc_id"))
161 if doc_id is None: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 return None
163 return _LibraryReference(
164 doc_id, path_match.group("suffix"), chunk_index=chunk_index
165 )
167 bracket_match = _BRACKETED_DOCUMENT_RE.match(candidate)
168 if bracket_match:
169 return _LibraryReference(
170 bracket_match.group("doc_id"),
171 bracketed=True,
172 chunk_index=chunk_index,
173 )
175 return None
178def parse_library_url(url: str) -> tuple[str, Optional[str]] | None:
179 """Return the local document reference and optional suffix, else ``None``.
181 ``suffix`` is ``"pdf"`` for the PDF-direct form, ``"chunks"`` for the
182 chunk-listing route that RAG citations point at, and ``None`` for the
183 root document page. Any ``#fragment`` (e.g. ``#chunk-3``) is stripped
184 before matching, so a chunk-targeted citation resolves to its
185 document. In addition to canonical library paths, this accepts
186 the ID-bound aliases emitted by the agent. The ``doc_id`` segment must
187 match the canonical document-ID pattern (32/64-hex or UUID), so
188 filename-shaped values like ``report.pdf`` are rejected here and fall
189 through to the egress-denial path; any accepted reference is still
190 scoped to the caller's own documents by ``get_user_db_session`` at
191 lookup time.
192 """
193 reference = _parse_library_reference(url)
194 if reference is None:
195 return None
196 return reference.value, reference.suffix
199def is_citation_reference(url: str) -> int | None:
200 """If *url* is a bare ``[N]`` citation marker, return ``N`` (int), else ``None``."""
201 if not isinstance(url, str):
202 return None
203 m = _CITATION_REF_RE.match(url.strip())
204 if not m:
205 return None
206 return int(m.group(1))
209def resolve_library_document(
210 url: str, username: Optional[str]
211) -> Optional[dict[str, Any]]:
212 """Look up a local library document by ``/library/document/<uuid>[/pdf]`` URL.
214 When the URL includes a chunk anchor (e.g. ``/library/document/<uuid>/chunks#chunk-3``),
215 retrieves the specific chunk text from ``DocumentChunk`` rather than the whole document.
216 If no chunk anchor is present, resolves to ``Document.text_content``. If a chunk anchor
217 is present but the chunk is missing, reports that the chunk was not found rather than
218 silently falling back to the whole document.
220 Returns a dict shaped like the fetch tool's success payload (``title``,
221 ``content``) so callers can pass it straight to ``_register_in_collector``
222 + the ``[N] Title:\\nURL:\\n\\n<body>`` formatter. The ``url`` field of
223 the dict is the *original* library URL, so a downstream re-citation still
224 reads "library/document/…" instead of something more confusing — except
225 for the bare ``[<uuid>]`` alias, which is not a URL at all and is
226 re-emitted as the resolved document's canonical route so it shares a
227 bibliography entry with the document's other citations.
229 Returns ``None`` for:
231 - URLs that don't match ``/library/document/...``
232 - Documents the user can't access (no ``username`` or no row found).
233 Callers should fall through to the egress policy / HTTP fetcher
234 for these so a malformed UUID produces the normal "blocked by
235 egress policy" outcome instead of a misleading custom error.
237 (Documents with empty text content return a result payload with
238 ``content=""``, which summary-mode handles as NOT RELEVANT without an
239 LLM call.)
240 """
241 reference = _parse_library_reference(url)
242 if reference is None:
243 return None
244 lookup_value = reference.value
245 chunk_index = reference.chunk_index
247 if not username:
248 # No user context → no library. Caller falls through to the
249 # egress policy, which rejects the URL as ``unsupported_scheme``.
250 return None
252 # Lazy imports: avoid pulling SQLAlchemy / user-DB modules into the
253 # module-load path for the (common) no-library case.
254 try:
255 from local_deep_research.database.models.library import (
256 Document,
257 DocumentChunk,
258 )
259 from local_deep_research.database.session_context import (
260 get_user_db_session,
261 )
262 except Exception:
263 logger.exception(
264 "library_resolver: failed to import DB modules for document "
265 "lookup (url={!r})",
266 url,
267 )
268 return None
270 try:
271 with get_user_db_session(username) as session:
272 query = session.query(Document)
273 document = query.filter_by(id=lookup_value).first()
274 if document is None:
275 document = query.filter_by(document_hash=lookup_value).first()
276 if document is None and re.fullmatch(
277 r"[0-9a-fA-F]{32}", lookup_value
278 ):
279 canonical_id = str(uuid.UUID(hex=lookup_value))
280 document = query.filter_by(id=canonical_id).first()
281 if document is None:
282 return None
284 chunk = None
285 if chunk_index is not None:
286 chunk = (
287 session.query(DocumentChunk)
288 .filter(
289 DocumentChunk.source_id == document.id,
290 DocumentChunk.chunk_index == chunk_index,
291 )
292 .order_by(DocumentChunk.collection_name, DocumentChunk.id)
293 .first()
294 )
295 if chunk is not None:
296 text = chunk.chunk_text or ""
297 else:
298 text = f"Chunk {chunk_index} not found for document {document.id}."
299 else:
300 text = document.text_content or ""
302 citation_url = url
303 if reference.bracketed:
304 # Built from the RESOLVED row's id, not from the alias
305 # value: the lookup above also accepts a document_hash and
306 # a dash-less UUID, neither of which is a live route.
307 resolved_id = str(document.id or "")
308 if is_safe_document_id(resolved_id): 308 ↛ 316line 308 didn't jump to line 316 because the condition on line 308 was always true
309 citation_url = f"/library/document/{resolved_id}"
310 else:
311 # Practically unreachable — ``Document.id`` is a UUID —
312 # but silence here leaves ``citation_url`` as the raw
313 # ``[uuid]`` alias, which is not a route: a dead link
314 # that also keys separately from the document's other
315 # citations. Log it rather than let it look intended.
316 logger.warning(
317 "library_resolver: resolved document id is not "
318 "route-safe, citation left as the bracketed alias "
319 "(id={!r})",
320 resolved_id,
321 )
322 return {
323 "title": document.title or f"Document {lookup_value}",
324 "content": text,
325 "url": citation_url,
326 # ``snippet`` is what the collector registers as the
327 # citation preview. Use the title fallback when text is
328 # empty so the agent still gets *something* descriptive.
329 "snippet": (
330 text[:200].strip() if text else document.title or ""
331 )
332 or "",
333 }
334 except Exception:
335 # Match the existing LibraryRAGSearchEngine behaviour: a per-document
336 # DB hiccup is non-fatal; the tool returns a clean "not found" via
337 # None and the caller can fall through.
338 logger.exception(
339 "library_resolver: failed to load document "
340 "(reference={!r}, username={!r})",
341 lookup_value,
342 username,
343 )
344 return None
347def make_library_resolver(
348 username: Optional[str],
349) -> Callable[[str], Optional[dict[str, Any]]]:
350 """Build a one-arg URL → dict resolver suitable for the fetch tool.
352 Captures *username* in the closure so subagent workers (which don't
353 inherit thread-local Flask session state) can resolve library URLs
354 using the run's user, the same way the LangGraph strategy threads
355 ``settings_snapshot`` and ``egress_context`` through tool closures.
357 Returns the resolved dict, or ``None`` for URLs that aren't library
358 document refs — the fetch tool then falls through to the egress gate
359 unchanged.
360 """
361 # Bind the username once so the inner function doesn't re-resolve on
362 # every call (cheap, but keeps the per-call hot path branch-free).
363 _username = username
365 def _resolve(url: str) -> Optional[dict[str, Any]]:
366 return resolve_library_document(url, _username)
368 return _resolve
371def resolve_citation_reference(
372 url: str,
373 collector: Any,
374) -> Optional[dict[str, Any]]:
375 """Resolve a bare ``[N]`` citation marker to its source result dict.
377 Returns the result dict (with at least ``link``/``url`` and ``title``
378 populated, mirroring ``SearchResultsCollector.find_by_index``) when
379 the marker matches a tracked citation, else ``None``. The fetch tool
380 uses ``None`` as a signal to return a helpful "no such citation"
381 error to the agent instead of falling through to the egress gate
382 (where the marker would also be rejected as ``unsupported_scheme``,
383 but with a less actionable message).
384 """
385 idx = is_citation_reference(url)
386 if idx is None:
387 return None
388 if collector is None: 388 ↛ 389line 388 didn't jump to line 389 because the condition on line 388 was never true
389 return None
390 find = getattr(collector, "find_by_index", None)
391 if not callable(find):
392 return None
393 return find(idx)