Coverage for src/local_deep_research/advanced_search_system/tools/fetch/library_resolver.py: 95%
141 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Resolve ``fetch_content`` URLs that aren't actually network URLs.
3The LangGraph agent's ``fetch_content`` tool receives two URL-shaped
4strings that the egress policy correctly rejects as
5``unsupported_scheme`` (and that the original ``ContentFetcher`` would
6not know how to fetch even without a policy gate):
81. **Local library document references** like
9 ``/library/document/<uuid>``, ``/library/document/<uuid>/chunks#chunk-3``,
10 ``/lib/document/<uuid>``, ``https://library.document/<uuid>``, or
11 ``[<uuid>]``.
12 The library RAG search engine and the collection search engine emit
13 these as the citation URL of a search hit (see
14 ``web_search_engines.engines.search_engine_library.LibraryRAGSearchEngine.search``
15 and ``...search_engine_collection.CollectionSearchEngine._get_document_url``).
16 The URL points at the user's own library — a local DB read, not a
17 network fetch — and resolving it via ``DocumentChunk.chunk_text`` (when
18 a chunk anchor is provided) or ``Document.text_content`` is what
19 the human user would see if they clicked the link in the UI.
212. **Bare citation markers** like ``[1062]``, ``[1084]``. The agent
22 sometimes pastes a citation marker it saw in the search-results block
23 back into the tool instead of the actual URL. ``SearchResultsCollector``
24 already maps URL → index; the inverse ``find_by_index`` lets the tool
25 resolve ``[N]`` back to its source.
27Both shapes are intercepted here BEFORE the egress policy gate, so the
28agent gets the real page content (or a helpful error) instead of a
29generic "blocked by egress policy" denial that wastes a fetch slot
30(A3 in ``research_f3045c5b_issue_analysis.md``).
32Both the path and bracket forms bind the document id to the canonical
3332/64-hex-or-UUID pattern, so a guessable filename (``report.pdf``) is
34rejected at the regex instead of relying on a downstream DB miss.
35Resolution always runs inside ``get_user_db_session(username)``, so a
36parsed reference can still only return a document the caller already owns.
37"""
39from __future__ import annotations
41import re
42import uuid
43from dataclasses import dataclass
44from typing import Any, Callable, Optional
45from urllib.parse import unquote, urlsplit
47from loguru import logger
49from ....utilities.chunk_anchor import (
50 LIBRARY_ROUTE_SUFFIX_ALTERNATION,
51 is_safe_document_id,
52)
53from ....utilities.url_utils import is_valid_chunk_fragment
55# Canonical document IDs are 32/64-hex blobs or a UUID. The path form is
56# bound to this same pattern as the bracket form (below) so a guessable
57# filename like ``/library/document/report.pdf`` is rejected at the regex
58# rather than reaching the DB only to miss.
59_DOCUMENT_ID_PATTERN = (
60 r"(?:"
61 r"[0-9a-fA-F]{32}|"
62 r"[0-9a-fA-F]{64}|"
63 r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
64 r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
65 r")"
66)
67# Match canonical paths and the ``/lib/document/...`` abbreviation observed
68# in agent tool calls. The same suffix is used by download_service.
69#
70# ``chunks`` is accepted alongside ``pdf`` because the library/collection
71# RAG engines cite chunk-targeted URLs (``/library/document/<id>/chunks``
72# plus a ``#chunk-<n>`` fragment) since #5381.
73_LIBRARY_PATH_RE = re.compile(
74 rf"^/(?:library|lib)/document/(?P<doc_id>{_DOCUMENT_ID_PATTERN})"
75 rf"(?:/(?P<suffix>{LIBRARY_ROUTE_SUFFIX_ALTERNATION}))?/?$"
76)
77_BRACKETED_DOCUMENT_RE = re.compile(
78 rf"^\[(?P<doc_id>{_DOCUMENT_ID_PATTERN})\]$"
79)
81# Match ``[N]`` — a 1-based citation marker with no surrounding chars.
82# The agent emits this verbatim when it confuses the citation marker for
83# a URL; we resolve it via SearchResultsCollector.find_by_index.
84_CITATION_REF_RE = re.compile(r"^\[(\d+)\]$")
87@dataclass(frozen=True)
88class _LibraryReference:
89 value: str
90 suffix: Optional[str] = None
91 #: True when the reference arrived as the bare ``[<uuid>]`` alias
92 #: rather than as a route. That spelling is not a URL at all, so
93 #: registering it verbatim as the citation link gives the document a
94 #: THIRD bibliography entry alongside its ``/library/document/<id>``
95 #: and ``https://library.document/<id>`` citations — and renders a
96 #: dead link. Callers re-emit the canonical route instead.
97 bracketed: bool = False
98 chunk_index: Optional[int] = None
101def _decode_segment(value: str) -> str | None:
102 """Percent-decode one path segment, refusing separators and controls.
104 Now that ``_LIBRARY_PATH_RE``'s doc_id group is bound to hex/UUID, the only
105 live call site can never hand this a ``%``, so ``unquote`` is a no-op there
106 and none of the checks below fire. It is kept deliberately rather than
107 inlined away: it is the guard that makes loosening that group safe, and
108 deleting it would silently remove the safety net for whoever does. Anyone
109 widening the doc_id charset should confirm this is reachable again.
110 """
111 decoded = unquote(value)
112 if ( 112 ↛ 117line 112 didn't jump to line 117 because the condition on line 112 was never true
113 not decoded
114 or any(char in decoded for char in ("/", "\\", "?", "#"))
115 or any(ord(char) < 32 or ord(char) == 127 for char in decoded)
116 ):
117 return None
118 return decoded
121def _parse_library_reference(url: str) -> _LibraryReference | None:
122 if not isinstance(url, str):
123 return None
125 candidate = url.strip()
126 if not candidate:
127 return None
129 # Extract any validated ``#chunk-<n>`` fragment before splitting the path.
130 # Chunk citations carry ``#chunk-<n>``, which addresses a specific chunk.
131 # We record the index and strip the fragment before the ``://`` test so that
132 # (a) the ``https://library.document/<id>/chunks#chunk-3`` alias is not
133 # rejected by the absolute branch's no-fragment rule, and (b) a
134 # relative path whose fragment happens to contain ``://``
135 # (``/library/document/<id>#http://x``) is not misrouted into it.
136 chunk_index: Optional[int] = None
137 if "#" in candidate:
138 head, fragment = candidate.split("#", 1)
139 candidate = head
140 if is_valid_chunk_fragment(fragment):
141 chunk_index = int(fragment[len("chunk-") :])
142 if not candidate:
143 return None
145 if "://" in candidate:
146 parsed = urlsplit(candidate)
147 # No ``parsed.fragment`` check: the fragment was stripped above, so
148 # it can never reach here. A query still disqualifies — it can carry
149 # server-meaningful state, unlike a client-side anchor.
150 if (
151 parsed.scheme != "https"
152 or parsed.netloc != "library.document"
153 or parsed.query
154 ):
155 return None
156 candidate = f"/library/document{parsed.path}"
158 path_match = _LIBRARY_PATH_RE.match(candidate)
159 if path_match:
160 doc_id = _decode_segment(path_match.group("doc_id"))
161 if doc_id is None: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 return None
163 return _LibraryReference(
164 doc_id, path_match.group("suffix"), chunk_index=chunk_index
165 )
167 bracket_match = _BRACKETED_DOCUMENT_RE.match(candidate)
168 if bracket_match:
169 return _LibraryReference(
170 bracket_match.group("doc_id"),
171 bracketed=True,
172 chunk_index=chunk_index,
173 )
175 return None
178def parse_library_url(url: str) -> tuple[str, Optional[str]] | None:
179 """Return the local document reference and optional suffix, else ``None``.
181 ``suffix`` is ``"pdf"`` for the PDF-direct form, ``"chunks"`` for the
182 chunk-listing route that RAG citations point at, and ``None`` for the
183 root document page. Any ``#fragment`` (e.g. ``#chunk-3``) is stripped
184 before matching, so a chunk-targeted citation resolves to its
185 document. In addition to canonical library paths, this accepts
186 the ID-bound aliases emitted by the agent. The ``doc_id`` segment must
187 match the canonical document-ID pattern (32/64-hex or UUID), so
188 filename-shaped values like ``report.pdf`` are rejected here and fall
189 through to the egress-denial path; any accepted reference is still
190 scoped to the caller's own documents by ``get_user_db_session`` at
191 lookup time.
192 """
193 reference = _parse_library_reference(url)
194 if reference is None:
195 return None
196 return reference.value, reference.suffix
199def is_citation_reference(url: str) -> int | None:
200 """If *url* is a bare ``[N]`` citation marker, return ``N`` (int), else ``None``."""
201 if not isinstance(url, str):
202 return None
203 m = _CITATION_REF_RE.match(url.strip())
204 if not m:
205 return None
206 return int(m.group(1))
209def resolve_library_document(
210 url: str, username: Optional[str], content_cache: Any = None
211) -> Optional[dict[str, Any]]:
212 """Look up a local library document by ``/library/document/<uuid>[/pdf]`` URL.
214 When the URL includes a chunk anchor (e.g. ``/library/document/<uuid>/chunks#chunk-3``),
215 retrieves the specific chunk text from ``DocumentChunk`` rather than the whole document.
216 If no chunk anchor is present, resolves to ``Document.text_content``. If a chunk anchor
217 is present but the chunk is missing, reports that the chunk was not found rather than
218 silently falling back to the whole document.
220 Returns a dict shaped like the fetch tool's success payload (``title``,
221 ``content``) so callers can pass it straight to ``_register_in_collector``
222 + the ``[N] Title:\\nURL:\\n\\n<body>`` formatter. The ``url`` field of
223 the dict is the *original* library URL, so a downstream re-citation still
224 reads "library/document/…" instead of something more confusing — except
225 for the bare ``[<uuid>]`` alias, which is not a URL at all and is
226 re-emitted as the resolved document's canonical route so it shares a
227 bibliography entry with the document's other citations.
229 Returns ``None`` for:
231 - URLs that don't match ``/library/document/...``
232 - Documents the user can't access (no ``username`` or no row found).
233 Callers should fall through to the egress policy / HTTP fetcher
234 for these so a malformed UUID produces the normal "blocked by
235 egress policy" outcome instead of a misleading custom error.
237 (Documents with empty text content return a result payload with
238 ``content=""``, which summary-mode handles as NOT RELEVANT without an
239 LLM call.)
241 When a shared ``content_cache`` is available (the single cache for the
242 whole research run), chunk / document text is served from it on repeat
243 reads — so the quick-report phase, detailed subsections, lead agent
244 and subagents do not re-query the same rows on the spot.
246 Only genuine library references consult the cache here. A cached web
247 page (e.g. ``https://example.com/page`` fetched via ``web_search``)
248 must never surface as a "library document": web URLs fall through to
249 the egress gate + budget + HTTP fetcher so accounting stays symmetric
250 (library reads are local and consume no network slot; web fetches do,
251 even on cache hits, preserving historical accounting).
252 """
253 reference = _parse_library_reference(url)
254 if reference is None:
255 return None
256 try:
257 from ....utilities.research_content_cache import (
258 resolve_cache as _resolve_cache,
259 )
261 _cache = _resolve_cache(content_cache)
262 except Exception: # pragma: no cover - defensive
263 _cache = None
264 if _cache is not None:
265 try:
266 _hit = _cache.get(url)
267 except Exception: # pragma: no cover - defensive
268 _hit = None
269 if _hit is not None and isinstance(_hit.get("content"), str):
270 _text = _hit["content"]
271 _title = _hit.get("title") or ""
272 _curl = _hit.get("url") or url
273 return {
274 "title": _title,
275 "content": _text,
276 "url": _curl,
277 "snippet": (_text[:200].strip() if _text else _title or "")
278 or "",
279 }
280 lookup_value = reference.value
281 chunk_index = reference.chunk_index
283 if not username:
284 # No user context → no library. Caller falls through to the
285 # egress policy, which rejects the URL as ``unsupported_scheme``.
286 return None
288 # Lazy imports: avoid pulling SQLAlchemy / user-DB modules into the
289 # module-load path for the (common) no-library case.
290 try:
291 from local_deep_research.database.models.library import (
292 Document,
293 DocumentChunk,
294 )
295 from local_deep_research.database.session_context import (
296 get_user_db_session,
297 )
298 except Exception:
299 logger.exception(
300 "library_resolver: failed to import DB modules for document "
301 "lookup (url={!r})",
302 url,
303 )
304 return None
306 try:
307 with get_user_db_session(username) as session:
308 query = session.query(Document)
309 document = query.filter_by(id=lookup_value).first()
310 if document is None:
311 document = query.filter_by(document_hash=lookup_value).first()
312 if document is None and re.fullmatch(
313 r"[0-9a-fA-F]{32}", lookup_value
314 ):
315 canonical_id = str(uuid.UUID(hex=lookup_value))
316 document = query.filter_by(id=canonical_id).first()
317 if document is None:
318 return None
320 chunk = None
321 if chunk_index is not None:
322 chunk = (
323 session.query(DocumentChunk)
324 .filter(
325 DocumentChunk.source_id == document.id,
326 DocumentChunk.chunk_index == chunk_index,
327 )
328 .order_by(DocumentChunk.collection_name, DocumentChunk.id)
329 .first()
330 )
331 if chunk is not None:
332 text = chunk.chunk_text or ""
333 else:
334 text = f"Chunk {chunk_index} not found for document {document.id}."
335 else:
336 text = document.text_content or ""
338 citation_url = url
339 if reference.bracketed:
340 # Built from the RESOLVED row's id, not from the alias
341 # value: the lookup above also accepts a document_hash and
342 # a dash-less UUID, neither of which is a live route.
343 resolved_id = str(document.id or "")
344 if is_safe_document_id(resolved_id): 344 ↛ 352line 344 didn't jump to line 352 because the condition on line 344 was always true
345 citation_url = f"/library/document/{resolved_id}"
346 else:
347 # Practically unreachable — ``Document.id`` is a UUID —
348 # but silence here leaves ``citation_url`` as the raw
349 # ``[uuid]`` alias, which is not a route: a dead link
350 # that also keys separately from the document's other
351 # citations. Log it rather than let it look intended.
352 logger.warning(
353 "library_resolver: resolved document id is not "
354 "route-safe, citation left as the bracketed alias "
355 "(id={!r})",
356 resolved_id,
357 )
358 # Skip the cache write when the requested chunk was not found
359 # (chunk_index is set but no DocumentChunk row matched).
360 # A later retry in the same run must hit the database rather
361 # than returning the stored "Chunk N not found" failure message.
362 if _cache is not None and not (
363 chunk_index is not None and chunk is None
364 ):
365 try:
366 _cache.put(
367 url,
368 text,
369 title=document.title or f"Document {lookup_value}",
370 url_out=citation_url,
371 )
372 except Exception: # allow: silent-exception
373 pass
374 return {
375 "title": document.title or f"Document {lookup_value}",
376 "content": text,
377 "url": citation_url,
378 # ``snippet`` is what the collector registers as the
379 # citation preview. Use the title fallback when text is
380 # empty so the agent still gets *something* descriptive.
381 "snippet": (
382 text[:200].strip() if text else document.title or ""
383 )
384 or "",
385 }
386 except Exception:
387 # Match the existing LibraryRAGSearchEngine behaviour: a per-document
388 # DB hiccup is non-fatal; the tool returns a clean "not found" via
389 # None and the caller can fall through.
390 logger.exception(
391 "library_resolver: failed to load document "
392 "(reference={!r}, username={!r})",
393 lookup_value,
394 username,
395 )
396 return None
399def make_library_resolver(
400 username: Optional[str],
401 content_cache: Any = None,
402) -> Callable[[str], Optional[dict[str, Any]]]:
403 """Build a one-arg URL → dict resolver suitable for the fetch tool.
405 Captures *username* in the closure so subagent workers (which don't
406 inherit thread-local Flask session state) can resolve library URLs
407 using the run's user, the same way the LangGraph strategy threads
408 ``settings_snapshot`` and ``egress_context`` through tool closures.
410 *content_cache* is the single shared cache for the whole research run;
411 when None, the cache bound to the current research (if any) is used.
413 Returns the resolved dict, or ``None`` for URLs that aren't library
414 document refs — the fetch tool then falls through to the egress gate
415 unchanged.
416 """
417 # Bind the username once so the inner function doesn't re-resolve on
418 # every call (cheap, but keeps the per-call hot path branch-free).
419 _username = username
420 _cache = content_cache
422 def _resolve(url: str) -> Optional[dict[str, Any]]:
423 if _cache is not None:
424 return resolve_library_document(
425 url, _username, content_cache=_cache
426 )
427 return resolve_library_document(url, _username)
429 return _resolve
432def resolve_citation_reference(
433 url: str,
434 collector: Any,
435) -> Optional[dict[str, Any]]:
436 """Resolve a bare ``[N]`` citation marker to its source result dict.
438 Returns the result dict (with at least ``link``/``url`` and ``title``
439 populated, mirroring ``SearchResultsCollector.find_by_index``) when
440 the marker matches a tracked citation, else ``None``. The fetch tool
441 uses ``None`` as a signal to return a helpful "no such citation"
442 error to the agent instead of falling through to the egress gate
443 (where the marker would also be rejected as ``unsupported_scheme``,
444 but with a less actionable message).
445 """
446 idx = is_citation_reference(url)
447 if idx is None:
448 return None
449 if collector is None: 449 ↛ 450line 449 didn't jump to line 450 because the condition on line 449 was never true
450 return None
451 find = getattr(collector, "find_by_index", None)
452 if not callable(find):
453 return None
454 return find(idx)