Coverage for src/local_deep_research/web/services/report_assembly_service.py: 95%
142 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Reconstruct the legacy "full report" view from structured storage.
3`research.report_content` stores only the synthesized answer (with inline
4`[N](url)` hyperlinks). Sources live in the `research_resources` table;
5metrics live in `research.research_meta`. This module rebuilds the
6combined `answer + ## Sources + ## Research Metrics` view on demand for
7display and export — chat reads `research.report_content` directly and
8never goes through this module.
10The legacy Sources guard expects the first non-blank entry to use the
11bracketed bibliography shape (`[N]`, grouped `[N, M]`, or historical `[]`)
12and to carry an indented `URL:` continuation line. Indexed entries mirror
13`text_optimization.citation_formatter._BIB_SOURCES_PATTERN`; the empty-index
14variant covers rows emitted before every source had an index. The currently
15dormant `advanced_search_system.findings.repository.format_links` instead
16emits `1. Title` entries, so it must not become a persisted Sources emitter
17without extending this guard and its round-trip tests.
19DO NOT write the assembled output back to `research.report_content` —
20that column must stay answer-only. Writing assembled output back would
21silently re-introduce the regex over-strip class of bugs that
22answer-only storage avoids.
23"""
25import json
26import re
27from typing import Any, Dict, List, Optional
29from loguru import logger
30from sqlalchemy.orm import Session
32from ...database.models.research import ResearchHistory, ResearchResource
33from ...text_optimization.citation_formatter import (
34 strip_appended_sources_sentinel,
35)
36from ...utilities.search_utilities import (
37 UNCITED_SOURCES_MODE_KEY,
38 UNCITED_SOURCES_MODES,
39 format_links_to_markdown,
40 resolve_uncited_sources_mode,
41)
42from ...utilities.url_utils import FETCHED_TITLE_KEY, canonical_url_key
44# Line-anchored regexes for the legacy-row guard. See `assemble_full_report`
45# for why a substring `in body` check is too loose. A Sources heading alone is
46# not enough: LLMs sometimes emit one followed by placeholder prose. Require
47# the first non-blank line to be a bracket-shaped legacy bibliography entry,
48# including grouped indices and the historical empty-index shape, and require
49# its URL continuation line so bracket-led prose cannot trip the guard.
50_LEGACY_SOURCES_RE = re.compile(
51 r"^## Sources\b[^\r\n]*\r?\n"
52 r"(?:[ \t]*\r?\n)*[ \t]*\[(?:\d+(?:[ \t]*,[ \t]*\d+)*)?\]"
53 r"[^\r\n]*\r?\n[ \t]+URL:[ \t]*\S",
54 re.MULTILINE,
55)
56_LEGACY_METRICS_RE = re.compile(r"^## Research Metrics\b", re.MULTILINE)
59def _current_user_uncited_mode(db_session: Session) -> Optional[str]:
60 """Read the requesting user's CURRENT saved uncited-sources mode.
62 Covers legacy rows that completed before snapshots were persisted
63 (chat/follow-up research created and finished before #6747): those
64 rows carry only ``{"submission": ...}`` forever, so the row snapshot
65 cannot decide the render. The requesting user's current preference is
66 the best available signal — it is what a re-run would generate with.
68 Returns the validated mode, or ``None`` when the setting is missing,
69 malformed, or unreadable (caller falls through to the thread context
70 and then ``"fallback"``). Never raises: rendering a report must not
71 crash on a preference read.
72 """
73 try:
74 from ...settings.manager import SettingsManager
76 raw = SettingsManager(db_session).get_setting(
77 UNCITED_SOURCES_MODE_KEY, None
78 )
79 except Exception:
80 logger.debug(
81 "report assembly: current uncited-sources mode unreadable; "
82 "falling back",
83 exc_info=True,
84 )
85 return None
86 if isinstance(raw, dict): 86 ↛ 87line 86 didn't jump to line 87 because the condition on line 86 was never true
87 raw = raw.get("value")
88 if isinstance(raw, str) and raw.strip() in UNCITED_SOURCES_MODES:
89 return raw.strip()
90 return None
93def _research_metadata(research) -> Dict[str, Any]:
94 """Read metadata from current rows and legacy JSON-string rows."""
95 try:
96 meta = getattr(research, "research_meta", None)
97 if isinstance(meta, str):
98 meta = json.loads(meta)
99 return meta if isinstance(meta, dict) else {}
100 except (json.JSONDecodeError, TypeError):
101 return {}
104def _research_settings_snapshot(research) -> Optional[Dict[str, Any]]:
105 """Return the saved ``settings_snapshot`` for a research row, if any.
107 Report/history routes run without a research-worker settings context,
108 so resolving ``report.uncited_sources_mode`` from the thread context
109 alone always yields ``fallback`` — even when the user saved
110 ``disabled`` or ``strict``. The snapshot captured at research start
111 (``research_meta["settings_snapshot"]``) is the applicable saved
112 preference and must be passed explicitly.
113 """
114 try:
115 snap = _research_metadata(research).get("settings_snapshot")
116 return snap if isinstance(snap, dict) else None
117 except Exception:
118 return None
121def assemble_full_report(
122 research: Optional[ResearchHistory],
123 db_session: Session,
124 uncited_mode: Optional[str] = None,
125 settings_snapshot: Optional[Dict[str, Any]] = None,
126) -> Optional[str]:
127 """Reconstruct the legacy report shape from structured storage.
129 Args:
130 research: The ResearchHistory ORM row. Must be loaded inside
131 the supplied ``db_session`` to avoid DetachedInstanceError
132 when accessing ``research.research_meta`` lazily.
133 db_session: Active SQLAlchemy session bound to the user DB.
134 Used to query ``research_resources`` for the sources block.
135 uncited_mode: Explicit ``report.uncited_sources_mode`` override.
136 When ``None`` (default), a saved value in the research row's
137 snapshot is honored. Legacy snapshots without this setting use
138 the requesting user's current preference.
139 settings_snapshot: Explicit settings snapshot override. It takes
140 precedence over the row only when it contains this setting.
142 Returns:
143 ``None`` when ``research`` is ``None`` (caller should map to
144 404). Otherwise assembled markdown: answer + optional
145 ``## Sources`` block + optional ``## Research Metrics`` block.
146 An existing row with no body / sources / metrics returns an
147 empty string ``""`` (a valid empty-but-found response).
148 """
149 if research is None:
150 return None
152 # Detailed reports saved in NO_HYPERLINKS mode, or after the citation
153 # formatter raised, before the save site removed it, still carry the
154 # report generator's appended-sources sentinel. It is an HTML comment,
155 # which renderMarkdown() shows as literal text, so it is removed before
156 # the body is displayed or exported.
157 body = strip_appended_sources_sentinel(research.report_content or "")
159 # Legacy-row guard: older rows already contain inline
160 # `## Sources` / `## Research Metrics` blocks in report_content. If
161 # we appended freshly-assembled sections to those rows we'd render
162 # the blocks twice. Match only at line-start to avoid false positives
163 # from prose that happens to contain the substring `## Sources` inline
164 # (e.g. an answer that quotes another markdown document).
165 has_legacy_sources = bool(_LEGACY_SOURCES_RE.search(body))
166 has_legacy_metrics = bool(_LEGACY_METRICS_RE.search(body))
168 parts = [body]
170 if not has_legacy_sources:
171 # Let any failure propagate: the callers already wrap this in a
172 # try/except that returns HTTP 500. Swallowing it here would emit a
173 # report that looks complete but is silently missing all sources.
174 sources_md = _build_sources_markdown(
175 research,
176 db_session,
177 body=body,
178 uncited_mode=uncited_mode,
179 settings_snapshot=settings_snapshot,
180 )
181 if sources_md:
182 parts.append("## Sources\n\n" + sources_md)
184 if not has_legacy_metrics:
185 metrics_md = _build_metrics_markdown(research)
186 if metrics_md:
187 parts.append("## Research Metrics\n" + metrics_md)
189 return "\n\n".join(parts)
192def _build_metrics_markdown(research: ResearchHistory) -> str:
193 """Render the Research Metrics block from persisted metadata.
195 Today the inline metrics block (research_service.py quick-summary
196 path) used ``results["iterations"]`` and a fresh save-time
197 timestamp. Both end up in ``research.research_meta`` (the save site
198 persists ``metadata["iterations"]`` and ``metadata["generated_at"]``)
199 so this read recovers the same values. Falls back to
200 ``research.completed_at`` for the timestamp when ``generated_at`` is
201 missing (legacy rows or scheduler-saved research).
203 Returns an empty string when nothing meaningful can be rendered.
204 """
205 meta = _research_metadata(research)
206 iterations = meta.get("iterations")
207 generated_at = meta.get("generated_at") or research.completed_at
208 lines = []
209 if iterations is not None:
210 lines.append(f"- Search Iterations: {iterations}")
211 if generated_at:
212 lines.append(f"- Generated at: {generated_at}")
213 return "\n".join(lines)
216def _build_sources_markdown(
217 research: ResearchHistory,
218 db_session: Session,
219 body: str = "",
220 uncited_mode: Optional[str] = None,
221 settings_snapshot: Optional[Dict[str, Any]] = None,
222) -> str:
223 """Render the Sources block from the ``research_resources`` table.
225 Maps each ResearchResource row back to the dict shape
226 ``format_links_to_markdown`` expects, preferring the original
227 citation index from ``resource_metadata['original_data']['index']``
228 (assigned by the search system at search time, and the number the
229 inline ``[N]`` references in the saved answer point to). Falls
230 back to row order when the original index was lost on save.
232 Only resources cited in ``body`` (the saved answer) are listed,
233 honoring ``report.uncited_sources_mode``. The saved snapshot on
234 the research row wins over the thread-local worker context (which
235 report/history routes never install): an explicit ``uncited_mode``
236 wins first, then an explicit snapshot containing this setting, then
237 the row's own snapshot containing it, then — for older rows that
238 lack this setting — the requesting user's current saved setting,
239 then the thread context.
240 """
241 resources = (
242 db_session.query(ResearchResource)
243 .filter_by(research_id=research.id)
244 .order_by(ResearchResource.id.asc())
245 .all()
246 )
248 all_links: List[Dict[str, Any]] = []
249 missing_index_count = 0
250 for fallback_idx, r in enumerate(resources, start=1):
251 # Defensive: legacy rows may have stored metadata as a string.
252 meta = (
253 r.resource_metadata if isinstance(r.resource_metadata, dict) else {}
254 )
255 original = (
256 meta.get("original_data")
257 if isinstance(meta.get("original_data"), dict)
258 else {}
259 )
260 # ``is None`` (not ``not``) so 0 isn't treated as missing.
261 index = original.get("index")
262 if index is None or index == "":
263 missing_index_count += 1
264 index = str(fallback_idx)
265 all_links.append(
266 {
267 "url": str(r.url) if r.url else "",
268 "title": str(r.title) if r.title else "Untitled",
269 "index": index,
270 "journal_quality": original.get("journal_quality"),
271 # A fetched title can predate an archive upgrade. Retain
272 # the capture metadata so the renderer still adds its date.
273 "source_type": original.get("source_type", r.source_type),
274 "original_url": original.get("original_url"),
275 "requested_url": original.get("requested_url"),
276 "timestamp": original.get("timestamp"),
277 "formatted_date": original.get("formatted_date"),
278 # ``title`` above is the column, which the search hit wrote.
279 # Rows saved before this key existed return None, which the
280 # renderer ignores.
281 FETCHED_TITLE_KEY: original.get(FETCHED_TITLE_KEY),
282 }
283 )
285 if missing_index_count:
286 # DEBUG (not WARNING): expected for legacy rows / URL-less
287 # entries skipped at save time. Render correctness is preserved
288 # via row-order fallback. Bind research_id so the message
289 # routes through the per-research log table.
290 logger.bind(research_id=research.id).debug(
291 "_build_sources_markdown: {} of {} rows missing original "
292 "citation index; using row order. Common cause: URL-less "
293 "entries were skipped at save time.",
294 missing_index_count,
295 len(resources),
296 )
298 return format_links_to_markdown(
299 all_links,
300 prose=body,
301 uncited_mode=_resolve_saved_uncited_mode(
302 research,
303 uncited_mode=uncited_mode,
304 settings_snapshot=settings_snapshot,
305 db_session=db_session,
306 ),
307 )
310def _resolve_saved_uncited_mode(
311 research,
312 uncited_mode: Optional[str] = None,
313 settings_snapshot: Optional[Dict[str, Any]] = None,
314 db_session: Optional[Session] = None,
315) -> str:
316 """Resolve the effective uncited-sources mode for a saved research row.
318 Precedence: explicit ``uncited_mode`` > explicit snapshot value >
319 the row's own saved snapshot value > the requesting user's current
320 saved setting > thread context (which report routes never install) >
321 ``"fallback"``. A snapshot without this key is not an override.
322 """
323 if uncited_mode is not None:
324 return resolve_uncited_sources_mode(uncited_mode)
325 if (
326 isinstance(settings_snapshot, dict)
327 and UNCITED_SOURCES_MODE_KEY in settings_snapshot
328 ):
329 return resolve_uncited_sources_mode(
330 None, settings_snapshot=settings_snapshot
331 )
332 row_snapshot = _research_settings_snapshot(research)
333 if row_snapshot is not None and UNCITED_SOURCES_MODE_KEY in row_snapshot:
334 return resolve_uncited_sources_mode(
335 None, settings_snapshot=row_snapshot
336 )
337 if db_session is not None: 337 ↛ 341line 337 didn't jump to line 341 because the condition on line 337 was always true
338 current = _current_user_uncited_mode(db_session)
339 if current is not None:
340 return current
341 return resolve_uncited_sources_mode(None, settings_snapshot=None)
344def _format_source_link(row: ResearchResource) -> Optional[Dict[str, str]]:
345 """Map one ``research_resources`` row to the news feed's link shape.
347 Shared by :func:`get_research_source_links` and its batched variant so
348 the two cannot drift. Returns ``None`` for a row the feed cannot link
349 to: a non-``http`` URL, or a ``url`` that is not a string at all.
350 Titles fall back to the bare domain when missing and are truncated to
351 50 chars, matching the existing list-card rendering in the news UI.
353 A non-``str`` ``url`` is skipped, not coerced — the same rule
354 ``count_distinct_sources`` and ``format_links_to_markdown`` already
355 apply, and for the same reason: the caller passes this result to
356 :func:`_dedup_key`, and ``canonical_url_key`` raises on a non-str
357 (it unpacks ``str.partition``). Coercing with ``str()`` instead would
358 render a Python repr as a clickable link. ``title`` gets the same
359 treatment, so a malformed row degrades to the domain fallback rather
360 than printing a repr onto a news card.
361 """
362 if not isinstance(row.url, str):
363 return None
364 url = row.url.strip()
365 if not url.startswith("http"):
366 return None
367 title = (row.title if isinstance(row.title, str) else "").strip()
368 if not title:
369 domain = url.split("//")[-1].split("/")[0]
370 title = domain.replace("www.", "")
371 if len(title) > 50: 371 ↛ 372line 371 didn't jump to line 372 because the condition on line 371 was never true
372 title = title[:50] + "..."
373 return {"url": url, "title": title}
376def _dedup_key(url: str) -> str:
377 """Canonical grouping key for a source link.
379 The SAME key ``format_links_to_markdown`` groups the bibliography by
380 and ``count_distinct_sources`` counts with, so a "top 3 sources" card,
381 the rendered ``## Sources`` block and the reported source count cannot
382 disagree about what counts as one source. Falls back to the raw URL if
383 canonicalization yields nothing, so an unrecognizable URL stays its
384 own source rather than merging with every other unrecognizable one.
385 """
386 return canonical_url_key(url) or url
389def get_research_source_links(
390 research_id: str, db_session: Session, limit: int = 3
391) -> List[Dict[str, str]]:
392 """Top-N DISTINCT source links for a research, in row-insertion order.
394 Returns dicts shaped ``{"url": str, "title": str}`` matching the
395 news feed's ``links`` contract (``news/api.py`` consumers). Titles
396 are domain-fallback when missing, truncated to 50 chars to match
397 the existing list-card rendering in the news UI.
399 Deduplicated on :func:`_dedup_key`, first occurrence winning, so
400 ``limit=N`` yields N *distinct* sources rather than N rows. One row
401 per source is not something the caller can assume: a search strategy
402 stores one ``research_resources`` row per piece of evidence, so the
403 same URL legitimately appears several times with different snippets
404 (see #5894), and without this a "top 3 sources" card could render one
405 URL three times. Because dedup happens in Python the row cap cannot
406 be pushed into SQL: the query fetches every matching row, but
407 formatting stops as soon as ``limit`` distinct sources are in hand.
409 Args:
410 research_id: The ResearchHistory id.
411 db_session: Active SQLAlchemy session bound to the user DB.
412 limit: Maximum number of DISTINCT links to return.
413 """
414 rows = (
415 db_session.query(ResearchResource)
416 .filter_by(research_id=research_id)
417 .filter(ResearchResource.url.isnot(None))
418 .order_by(ResearchResource.id.asc())
419 )
420 out: List[Dict[str, str]] = []
421 seen: set[str] = set()
422 for r in rows:
423 if len(out) >= limit:
424 break
425 link = _format_source_link(r)
426 if link is None:
427 continue
428 key = _dedup_key(link["url"])
429 if key in seen:
430 continue
431 seen.add(key)
432 out.append(link)
433 return out
436def get_research_source_links_batch(
437 research_ids: List[str], db_session: Session, limit: Optional[int] = 3
438) -> Dict[str, List[Dict[str, str]]]:
439 """Batched variant of :func:`get_research_source_links`.
441 For news-feed list views that would otherwise fire one query per
442 research item (N+1). One ``WHERE research_id IN (...)`` query plus
443 Python-side grouping. Returned dict maps each research_id to its
444 top-N links (same shape as :func:`get_research_source_links`, and
445 deduplicated the same way — ``limit`` counts DISTINCT sources, not
446 rows). Research ids with zero rows map to ``[]``.
448 ``limit=None`` returns every link for each research (no cap) — used by
449 the report API, which exposes the full source list rather than a top-N.
450 """
451 result: Dict[str, List[Dict[str, str]]] = {rid: [] for rid in research_ids}
452 if not research_ids:
453 return result
455 rows = (
456 db_session.query(ResearchResource)
457 .filter(ResearchResource.research_id.in_(research_ids))
458 .filter(ResearchResource.url.isnot(None))
459 .order_by(ResearchResource.research_id, ResearchResource.id.asc())
460 .all()
461 )
462 seen_by_research: Dict[str, set[str]] = {}
463 for r in rows:
464 bucket = result.setdefault(r.research_id, [])
465 if limit is not None and len(bucket) >= limit:
466 continue
467 link = _format_source_link(r)
468 if link is None:
469 continue
470 seen = seen_by_research.setdefault(r.research_id, set())
471 key = _dedup_key(link["url"])
472 if key in seen:
473 continue
474 seen.add(key)
475 bucket.append(link)
476 return result