Coverage for src/local_deep_research/web/services/report_assembly_service.py: 99%

1"""Reconstruct the legacy "full report" view from structured storage. 

2 

3`research.report_content` stores only the synthesized answer (with inline 

4`[N](url)` hyperlinks). Sources live in the `research_resources` table; 

5metrics live in `research.research_meta`. This module rebuilds the 

6combined `answer + ## Sources + ## Research Metrics` view on demand for 

7display and export — chat reads `research.report_content` directly and 

8never goes through this module. 

9 

10The legacy Sources guard expects the first non-blank entry to use the 

11bracketed bibliography shape (`[N]`, grouped `[N, M]`, or historical `[]`) 

12and to carry an indented `URL:` continuation line. Indexed entries mirror 

13`text_optimization.citation_formatter._BIB_SOURCES_PATTERN`; the empty-index 

14variant covers rows emitted before every source had an index. The currently 

15dormant `advanced_search_system.findings.repository.format_links` instead 

16emits `1. Title` entries, so it must not become a persisted Sources emitter 

17without extending this guard and its round-trip tests. 

18 

19DO NOT write the assembled output back to `research.report_content` — 

20that column must stay answer-only. Writing assembled output back would 

21silently re-introduce the regex over-strip class of bugs that 

22answer-only storage avoids. 

23""" 

24 

25import re 

26from typing import Any, Dict, List, Optional 

27 

28from loguru import logger 

29from sqlalchemy.orm import Session 

30 

31from ...database.models.research import ResearchHistory, ResearchResource 

32from ...utilities.search_utilities import format_links_to_markdown 

33from ...utilities.url_utils import canonical_url_key 

34 

35# Line-anchored regexes for the legacy-row guard. See `assemble_full_report` 

36# for why a substring `in body` check is too loose. A Sources heading alone is 

37# not enough: LLMs sometimes emit one followed by placeholder prose. Require 

38# the first non-blank line to be a bracket-shaped legacy bibliography entry, 

39# including grouped indices and the historical empty-index shape, and require 

40# its URL continuation line so bracket-led prose cannot trip the guard. 

41_LEGACY_SOURCES_RE = re.compile( 

42 r"^## Sources\b[^\r\n]*\r?\n" 

43 r"(?:[ \t]*\r?\n)*[ \t]*\[(?:\d+(?:[ \t]*,[ \t]*\d+)*)?\]" 

44 r"[^\r\n]*\r?\n[ \t]+URL:[ \t]*\S", 

45 re.MULTILINE, 

46) 

47_LEGACY_METRICS_RE = re.compile(r"^## Research Metrics\b", re.MULTILINE) 

48 

49 

50def assemble_full_report( 

51 research: Optional[ResearchHistory], db_session: Session 

52) -> Optional[str]: 

53 """Reconstruct the legacy report shape from structured storage. 

54 

55 Args: 

56 research: The ResearchHistory ORM row. Must be loaded inside 

57 the supplied ``db_session`` to avoid DetachedInstanceError 

58 when accessing ``research.research_meta`` lazily. 

59 db_session: Active SQLAlchemy session bound to the user DB. 

60 Used to query ``research_resources`` for the sources block. 

61 

62 Returns: 

63 ``None`` when ``research`` is ``None`` (caller should map to 

64 404). Otherwise assembled markdown: answer + optional 

65 ``## Sources`` block + optional ``## Research Metrics`` block. 

66 An existing row with no body / sources / metrics returns an 

67 empty string ``""`` (a valid empty-but-found response). 

68 """ 

69 if research is None: 

70 return None 

71 

72 body = research.report_content or "" 

73 

74 # Legacy-row guard: older rows already contain inline 

75 # `## Sources` / `## Research Metrics` blocks in report_content. If 

76 # we appended freshly-assembled sections to those rows we'd render 

77 # the blocks twice. Match only at line-start to avoid false positives 

78 # from prose that happens to contain the substring `## Sources` inline 

79 # (e.g. an answer that quotes another markdown document). 

80 has_legacy_sources = bool(_LEGACY_SOURCES_RE.search(body)) 

81 has_legacy_metrics = bool(_LEGACY_METRICS_RE.search(body)) 

82 

83 parts = [body] 

84 

85 if not has_legacy_sources: 

86 # Let any failure propagate: the callers already wrap this in a 

87 # try/except that returns HTTP 500. Swallowing it here would emit a 

88 # report that looks complete but is silently missing all sources. 

89 sources_md = _build_sources_markdown(research, db_session) 

90 if sources_md: 

91 parts.append("## Sources\n\n" + sources_md) 

92 

93 if not has_legacy_metrics: 

94 metrics_md = _build_metrics_markdown(research) 

95 if metrics_md: 

96 parts.append("## Research Metrics\n" + metrics_md) 

97 

98 return "\n\n".join(parts) 

99 

100 

101def _build_metrics_markdown(research: ResearchHistory) -> str: 

102 """Render the Research Metrics block from persisted metadata. 

103 

104 Today the inline metrics block (research_service.py quick-summary 

105 path) used ``results["iterations"]`` and a fresh save-time 

106 timestamp. Both end up in ``research.research_meta`` (the save site 

107 persists ``metadata["iterations"]`` and ``metadata["generated_at"]``) 

108 so this read recovers the same values. Falls back to 

109 ``research.completed_at`` for the timestamp when ``generated_at`` is 

110 missing (legacy rows or scheduler-saved research). 

111 

112 Returns an empty string when nothing meaningful can be rendered. 

113 """ 

114 meta = research.research_meta or {} 

115 iterations = meta.get("iterations") 

116 generated_at = meta.get("generated_at") or research.completed_at 

117 lines = [] 

118 if iterations is not None: 

119 lines.append(f"- Search Iterations: {iterations}") 

120 if generated_at: 

121 lines.append(f"- Generated at: {generated_at}") 

122 return "\n".join(lines) 

123 

124 

125def _build_sources_markdown( 

126 research: ResearchHistory, db_session: Session 

127) -> str: 

128 """Render the Sources block from the ``research_resources`` table. 

129 

130 Maps each ResearchResource row back to the dict shape 

131 ``format_links_to_markdown`` expects, preferring the original 

132 citation index from ``resource_metadata['original_data']['index']`` 

133 (assigned by the search system at search time, and the number the 

134 inline ``[N]`` references in the saved answer point to). Falls 

135 back to row order when the original index was lost on save. 

136 """ 

137 resources = ( 

138 db_session.query(ResearchResource) 

139 .filter_by(research_id=research.id) 

140 .order_by(ResearchResource.id.asc()) 

141 .all() 

142 ) 

143 

144 all_links: List[Dict[str, Any]] = [] 

145 missing_index_count = 0 

146 for fallback_idx, r in enumerate(resources, start=1): 

147 # Defensive: legacy rows may have stored metadata as a string. 

148 meta = ( 

149 r.resource_metadata if isinstance(r.resource_metadata, dict) else {} 

150 ) 

151 original = ( 

152 meta.get("original_data") 

153 if isinstance(meta.get("original_data"), dict) 

154 else {} 

155 ) 

156 # ``is None`` (not ``not``) so 0 isn't treated as missing. 

157 index = original.get("index") 

158 if index is None or index == "": 

159 missing_index_count += 1 

160 index = str(fallback_idx) 

161 all_links.append( 

162 { 

163 "url": str(r.url) if r.url else "", 

164 "title": str(r.title) if r.title else "Untitled", 

165 "index": index, 

166 "journal_quality": original.get("journal_quality"), 

167 } 

168 ) 

169 

170 if missing_index_count: 

171 # DEBUG (not WARNING): expected for legacy rows / URL-less 

172 # entries skipped at save time. Render correctness is preserved 

173 # via row-order fallback. Bind research_id so the message 

174 # routes through the per-research log table. 

175 logger.bind(research_id=research.id).debug( 

176 "_build_sources_markdown: {} of {} rows missing original " 

177 "citation index; using row order. Common cause: URL-less " 

178 "entries were skipped at save time.", 

179 missing_index_count, 

180 len(resources), 

181 ) 

182 

183 return format_links_to_markdown(all_links) 

184 

185 

186def _format_source_link(row: ResearchResource) -> Optional[Dict[str, str]]: 

187 """Map one ``research_resources`` row to the news feed's link shape. 

188 

189 Shared by :func:`get_research_source_links` and its batched variant so 

190 the two cannot drift. Returns ``None`` for a row the feed cannot link 

191 to: a non-``http`` URL, or a ``url`` that is not a string at all. 

192 Titles fall back to the bare domain when missing and are truncated to 

193 50 chars, matching the existing list-card rendering in the news UI. 

194 

195 A non-``str`` ``url`` is skipped, not coerced — the same rule 

196 ``count_distinct_sources`` and ``format_links_to_markdown`` already 

197 apply, and for the same reason: the caller passes this result to 

198 :func:`_dedup_key`, and ``canonical_url_key`` raises on a non-str 

199 (it unpacks ``str.partition``). Coercing with ``str()`` instead would 

200 render a Python repr as a clickable link. ``title`` gets the same 

201 treatment, so a malformed row degrades to the domain fallback rather 

202 than printing a repr onto a news card. 

203 """ 

204 if not isinstance(row.url, str): 

205 return None 

206 url = row.url.strip() 

207 if not url.startswith("http"): 

208 return None 

209 title = (row.title if isinstance(row.title, str) else "").strip() 

210 if not title: 

211 domain = url.split("//")[-1].split("/")[0] 

212 title = domain.replace("www.", "") 

213 if len(title) > 50: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true

214 title = title[:50] + "..." 

215 return {"url": url, "title": title} 

216 

217 

218def _dedup_key(url: str) -> str: 

219 """Canonical grouping key for a source link. 

220 

221 The SAME key ``format_links_to_markdown`` groups the bibliography by 

222 and ``count_distinct_sources`` counts with, so a "top 3 sources" card, 

223 the rendered ``## Sources`` block and the reported source count cannot 

224 disagree about what counts as one source. Falls back to the raw URL if 

225 canonicalization yields nothing, so an unrecognizable URL stays its 

226 own source rather than merging with every other unrecognizable one. 

227 """ 

228 return canonical_url_key(url) or url 

229 

230 

231def get_research_source_links( 

232 research_id: str, db_session: Session, limit: int = 3 

233) -> List[Dict[str, str]]: 

234 """Top-N DISTINCT source links for a research, in row-insertion order. 

235 

236 Returns dicts shaped ``{"url": str, "title": str}`` matching the 

237 news feed's ``links`` contract (``news/api.py`` consumers). Titles 

238 are domain-fallback when missing, truncated to 50 chars to match 

239 the existing list-card rendering in the news UI. 

240 

241 Deduplicated on :func:`_dedup_key`, first occurrence winning, so 

242 ``limit=N`` yields N *distinct* sources rather than N rows. One row 

243 per source is not something the caller can assume: a search strategy 

244 stores one ``research_resources`` row per piece of evidence, so the 

245 same URL legitimately appears several times with different snippets 

246 (see #5894), and without this a "top 3 sources" card could render one 

247 URL three times. Because dedup happens in Python the row cap cannot 

248 be pushed into SQL: the query fetches every matching row, but 

249 formatting stops as soon as ``limit`` distinct sources are in hand. 

250 

251 Args: 

252 research_id: The ResearchHistory id. 

253 db_session: Active SQLAlchemy session bound to the user DB. 

254 limit: Maximum number of DISTINCT links to return. 

255 """ 

256 rows = ( 

257 db_session.query(ResearchResource) 

258 .filter_by(research_id=research_id) 

259 .filter(ResearchResource.url.isnot(None)) 

260 .order_by(ResearchResource.id.asc()) 

261 ) 

262 out: List[Dict[str, str]] = [] 

263 seen: set[str] = set() 

264 for r in rows: 

265 if len(out) >= limit: 

266 break 

267 link = _format_source_link(r) 

268 if link is None: 

269 continue 

270 key = _dedup_key(link["url"]) 

271 if key in seen: 

272 continue 

273 seen.add(key) 

274 out.append(link) 

275 return out 

276 

277 

278def get_research_source_links_batch( 

279 research_ids: List[str], db_session: Session, limit: Optional[int] = 3 

280) -> Dict[str, List[Dict[str, str]]]: 

281 """Batched variant of :func:`get_research_source_links`. 

282 

283 For news-feed list views that would otherwise fire one query per 

284 research item (N+1). One ``WHERE research_id IN (...)`` query plus 

285 Python-side grouping. Returned dict maps each research_id to its 

286 top-N links (same shape as :func:`get_research_source_links`, and 

287 deduplicated the same way — ``limit`` counts DISTINCT sources, not 

288 rows). Research ids with zero rows map to ``[]``. 

289 

290 ``limit=None`` returns every link for each research (no cap) — used by 

291 the report API, which exposes the full source list rather than a top-N. 

292 """ 

293 result: Dict[str, List[Dict[str, str]]] = {rid: [] for rid in research_ids} 

294 if not research_ids: 

295 return result 

296 

297 rows = ( 

298 db_session.query(ResearchResource) 

299 .filter(ResearchResource.research_id.in_(research_ids)) 

300 .filter(ResearchResource.url.isnot(None)) 

301 .order_by(ResearchResource.research_id, ResearchResource.id.asc()) 

302 .all() 

303 ) 

304 seen_by_research: Dict[str, set[str]] = {} 

305 for r in rows: 

306 bucket = result.setdefault(r.research_id, []) 

307 if limit is not None and len(bucket) >= limit: 

308 continue 

309 link = _format_source_link(r) 

310 if link is None: 

311 continue 

312 seen = seen_by_research.setdefault(r.research_id, set()) 

313 key = _dedup_key(link["url"]) 

314 if key in seen: 

315 continue 

316 seen.add(key) 

317 bucket.append(link) 

318 return result