Coverage for src/local_deep_research/report_generator.py: 88%
462 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1import importlib
2import re
3from typing import Any, Dict, List, Optional
4from datetime import datetime, UTC
6from langchain_core.language_models import BaseChatModel
7from loguru import logger
9# Fix circular import by importing directly from source modules
10from .config.llm_config import get_llm
11from .config.thread_settings import get_setting_from_snapshot
12from .search_system import AdvancedSearchSystem
13from .text_optimization.citation_formatter import (
14 LDR_APPENDED_SOURCES_SENTINEL,
15)
16from .utilities.json_utils import get_llm_response_text
18# Default constants for context accumulation to avoid repetition
19# These are used as fallbacks when settings are not available
20DEFAULT_MAX_CONTEXT_SECTIONS = (
21 3 # Number of previous sections to include as context
22)
23DEFAULT_MAX_CONTEXT_CHARS = (
24 4000 # Max characters for context (safe for smaller local models)
25)
27# Spelled-out output rules appended to every per-subsection prompt. Small
28# and/or quantized local models routinely ignore long, vague instructions,
29# so this is intentionally short, numbered, and refers to the actual
30# rendering consequences (a duplicate heading, a duplicate bibliography)
31# rather than abstract guidance.
32#
33# The framework always (a) inserts `## i.j Name` itself before appending
34# LLM content and (b) appends one consolidated `## Sources` block to the
35# whole report at the end of _format_final_report. The LLM does not need
36# to repeat either of those things, and historically has — producing
37# visibly stacked duplicate headings and duplicate "## Sources" sections
38# per subsection.
39_SUBSECTION_OUTPUT_GUIDANCE = (
40 "\n\nOUTPUT FORMAT RULES (FOLLOW EXACTLY):\n"
41 "1. Do NOT start your output with a Markdown heading of any level "
42 "(#, ##, ###, ####, etc.). The framework already inserts the "
43 "subsection heading for you; starting with your own heading line "
44 "creates a visible duplicate next to it in the final report.\n"
45 "2. Do NOT end your output with a '## Sources', '## References', "
46 "'## Bibliography', '## Citations', '## Key References', or "
47 "'## Selected Bibliography' section. The framework appends a single "
48 "consolidated '## Sources' block to the entire report after every "
49 "subsection is written; including your own bibliography duplicates "
50 "the same source list.\n"
51 "3. Begin your output with prose (a paragraph or a table), not a "
52 "heading and not an italic purpose statement. You may use '###' / "
53 "'####' / deeper levels for internal sub-subheadings inside this "
54 "subsection only."
55)
57# Pre-compiled normalisers used by _strip_subsection_boilerplate. These
58# are best-effort cleanups that run on the LLM's raw `current_knowledge`
59# before it is appended to the section — they do NOT touch the
60# framework's own headings or its trailing '## Sources' block.
61#
62# Leading heading: any ATX heading (level 1-6) at the very start of the
63# subsection content. Limited to a single match so we never strip a real
64# sub-subheading used to organise the body. Real reports show the LLM
65# almost always opens with a redundant ``### <subsection name>`` (or a
66# sibling's name, or a Roman-numeral-prefixed restatement) immediately
67# under the framework's own ``## i.j Name`` heading.
68_LEADING_HEADING_RE = re.compile(
69 r"\A[ \t\n]*[ \t]{0,3}#{1,6}[ \t]+[^\n]*",
70)
72# Leading italic purpose statement that mirrors the framework's
73# ``_<purpose>_`` subtitle (e.g. ``_To summarise the section's scope..._``).
74# Supports single underscores (_..._) or single asterisks (*...*).
75# Limited to a single match at the start.
76_LEADING_ITALIC_PURPOSE_RE = re.compile(
77 r"\A[ \t\n]*(?:_[^_\n]+_|(?:\*(?!\*)[^*\n]+\*))[ \t]*",
78)
80_BIB_KEYWORD_PATTERN = (
81 r"(?:"
82 r"Sources?|References?|Bibliograph(?:y|ies)|Citations?|"
83 r"Key[ \t]+References?|Selected[ \t]+Bibliograph(?:y|ies)|"
84 r"Works?[ \t]+Cited|Cited[ \t]+Works?|"
85 r"Additional[ \t]+Resources?|Further[ \t]+Reading"
86 r")"
87)
89_BIB_HEADINGS_CHAIN = (
90 rf"{_BIB_KEYWORD_PATTERN}"
91 rf"(?:[ \t]*(?:,|and|&|/)[ \t]+{_BIB_KEYWORD_PATTERN})*"
92)
94_BIB_QUALIFIER = (
95 r"(?:"
96 r"[ \t]+for[ \t]+[^\n]+"
97 r"|[ \t]*\([^\n]+\)"
98 r"|[ \t]*[:\-–—][ \t]*[^\n]+"
99 r")?"
100)
102# Bibliography-style heading at levels 1-6. Used to locate the start of a
103# per-subsection sources block that the framework already consolidates at
104# the end of the whole report. The closed label grammar ensures substantive
105# headings like "### Sources and Methods" or "## Reference Architecture" are preserved.
106_BIBLIOGRAPHY_HEADING_RE = re.compile(
107 rf"(?m)^[ \t]{{0,3}}#{{1,6}}[ \t]+"
108 rf"{_BIB_HEADINGS_CHAIN}"
109 rf"{_BIB_QUALIFIER}"
110 r"[ \t]*$",
111 re.IGNORECASE,
112)
114# Next ATX heading at levels 1-6 — marks the end of a bibliography block
115# when more subsection content follows it (common when the LLM appends a
116# "Selected Bibliography" mid-stream and then continues writing).
117_NEXT_HEADING_RE = re.compile(r"(?m)^[ \t]{0,3}#{1,6}[ \t]+")
119# Decorative horizontal rules the LLM often wraps around bibliography
120# blocks (``---`` on its own line).
121_HR_LINE_RE = re.compile(r"(?m)^[ \t]*-{3,}[ \t]*\n?")
123# Citation-list shape: digit+dot, bullet, or bracket citation start
124_CITATION_LINE_RE = re.compile(
125 r"^\s*(?:\d+[\.\)]\s+|[-*•]\s+|\[\d|\[cite|\bhttps?://|\bdoi:)",
126 re.IGNORECASE,
127)
129# Italic note line matching LLM boilerplate (e.g. "*(Note: ... bibliography ...)*")
130_BIB_NOTE_RE = re.compile(
131 r"^\s*[\*_]\s*\(?Note\b[^\n]*?\b(?:bibliography|sources|references|citations)\b",
132 re.IGNORECASE,
133)
136def _get_code_fence_spans(text: str) -> List[tuple[int, int]]:
137 """Return character index ranges (start, end) that are inside code blocks.
139 Recognizes:
140 - Backtick fenced blocks (``` or ```` etc.), paired by delimiter type and length.
141 - Tilde fenced blocks (~~~ or ~~~~ etc.), paired by delimiter type and length.
142 - Indented code blocks (lines indented by 4+ spaces or tabs, preceded by a blank line).
143 """
144 spans: List[tuple[int, int]] = []
145 if not text: 145 ↛ 146line 145 didn't jump to line 146 because the condition on line 145 was never true
146 return spans
148 lines = text.splitlines(keepends=True)
149 in_fence = False
150 fence_char = ""
151 fence_len = 0
152 fence_start = 0
154 in_indented = False
155 indented_start = 0
156 last_indented_end = 0
157 prev_blank = True
159 line_offset = 0
160 fence_open_re = re.compile(r"^[ \t]{0,3}(`{3,}|~{3,})")
162 for line in lines:
163 line_start = line_offset
164 line_end = line_offset + len(line)
165 line_offset = line_end
167 stripped = line.rstrip("\r\n")
169 if in_fence:
170 close_re = (
171 rf"^[ \t]{{0,3}}{re.escape(fence_char)}{{{fence_len},}}[ \t]*$"
172 )
173 if re.match(close_re, stripped):
174 in_fence = False
175 spans.append((fence_start, line_end))
176 continue
178 open_match = fence_open_re.match(stripped)
179 if open_match:
180 if in_indented: 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true
181 in_indented = False
182 spans.append((indented_start, last_indented_end))
184 in_fence = True
185 delim = open_match.group(1)
186 fence_char = delim[0]
187 fence_len = len(delim)
188 fence_start = line_start
189 prev_blank = False
190 continue
192 is_indented = stripped.startswith(" ") or stripped.startswith("\t")
193 is_blank = not stripped.strip()
195 if in_indented:
196 if is_indented:
197 last_indented_end = line_end
198 elif is_blank:
199 pass
200 else:
201 in_indented = False
202 spans.append((indented_start, last_indented_end))
203 else:
204 if is_indented and prev_blank:
205 in_indented = True
206 indented_start = line_start
207 last_indented_end = line_end
209 prev_blank = is_blank
211 if in_fence: 211 ↛ 212line 211 didn't jump to line 212 because the condition on line 211 was never true
212 spans.append((fence_start, len(text)))
213 elif in_indented: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true
214 spans.append((indented_start, last_indented_end))
216 return spans
219def _is_in_spans(pos: int, spans: List[tuple[int, int]]) -> bool:
220 """Return True if pos falls within any range in spans."""
221 return any(start <= pos < end for start, end in spans)
224def get_report_generator(search_system=None):
225 """Return an instance of the report generator with default settings.
227 Args:
228 search_system: Optional existing AdvancedSearchSystem to use
229 """
230 return IntegratedReportGenerator(search_system=search_system)
233class IntegratedReportGenerator:
234 def __init__(
235 self,
236 searches_per_section: int = 2,
237 search_system=None,
238 llm: BaseChatModel | None = None,
239 settings_snapshot: Optional[Dict] = None,
240 ):
241 """
242 Args:
243 searches_per_section: Number of searches to perform for each
244 section in the report.
245 search_system: Custom search system to use, otherwise just uses
246 the default.
247 llm: Custom LLM to use. Required if search_system is not provided.
248 settings_snapshot: Optional settings snapshot for configurable values.
250 """
251 # If search_system is provided, use its LLM; otherwise use the provided LLM
252 self._owns_llm = False
253 if search_system:
254 self.search_system = search_system
255 self.model = llm or search_system.model
256 elif llm:
257 self.model = llm
258 self.search_system = AdvancedSearchSystem(llm=self.model) # type: ignore[call-arg]
259 else:
260 # Fallback for backwards compatibility - will only work with auth
261 self._owns_llm = True
262 self.model = get_llm()
263 self.search_system = AdvancedSearchSystem(llm=self.model) # type: ignore[call-arg]
265 self.searches_per_section = (
266 searches_per_section # Control search depth per section
267 )
269 # Load context settings from snapshot or use defaults
270 self.max_context_sections = get_setting_from_snapshot(
271 "report.max_context_sections",
272 default=DEFAULT_MAX_CONTEXT_SECTIONS,
273 settings_snapshot=settings_snapshot,
274 )
275 self.max_context_chars = get_setting_from_snapshot(
276 "report.max_context_chars",
277 default=DEFAULT_MAX_CONTEXT_CHARS,
278 settings_snapshot=settings_snapshot,
279 )
281 def close(self) -> None:
282 """Close the LLM client if this instance created it."""
283 from .utilities.resource_utils import safe_close
285 if self._owns_llm:
286 safe_close(self.model, "report generator LLM")
288 def generate_report(
289 self,
290 initial_findings: Dict,
291 query: str,
292 progress_callback=None,
293 ) -> Dict:
294 """Generate a complete research report with section-specific research.
296 Args:
297 initial_findings: Results from initial research phase.
298 query: Original user query.
299 progress_callback: Optional callable(message, progress_percent, metadata)
300 for reporting progress (0-100%) and checking cancellation.
301 """
303 # Step 1: Determine structure
304 if progress_callback:
305 progress_callback(
306 "Determining report structure",
307 0,
308 {"phase": "report_structure"},
309 )
310 structure = self._determine_report_structure(initial_findings, query)
312 # Step 2: Research and generate content for each section in one step
313 sections = self._research_and_generate_sections(
314 initial_findings,
315 structure,
316 query,
317 progress_callback=progress_callback,
318 )
320 # Step 3: Format final report
321 if progress_callback:
322 progress_callback(
323 "Formatting final report",
324 90,
325 {"phase": "report_formatting"},
326 )
327 report = self._format_final_report(sections, structure, query)
329 if progress_callback:
330 progress_callback(
331 "Report complete", 100, {"phase": "report_complete"}
332 )
334 return report
336 def _determine_report_structure(
337 self, findings: Dict, query: str
338 ) -> List[Dict]:
339 """Analyze content and determine optimal report structure."""
340 combined_content = findings["current_knowledge"]
341 prompt = f"""
342 Analyze this research content about: {query}
344 Content Summary:
345 {combined_content[:1000]}... [truncated]
347 Determine the most appropriate report structure by:
348 1. Analyzing the type of content (technical, business, academic, etc.)
349 2. Identifying main themes and logical groupings
350 3. Considering the depth and breadth of the research
352 Return a table of contents structure in this exact format:
353 STRUCTURE
354 1. [Section Name]
355 - [Subsection] | [purpose]
356 2. [Section Name]
357 - [Subsection] | [purpose]
358 ...
359 END_STRUCTURE
361 Make the structure specific to the content, not generic.
362 Each subsection must include its purpose after the | symbol.
363 DO NOT include sections about sources, citations, references, or methodology.
364 """
366 response = get_llm_response_text(self.model.invoke(prompt))
368 # Parse the structure
369 structure: List[Dict[str, Any]] = []
370 current_section: Optional[Dict[str, Any]] = None
372 for line in response.split("\n"):
373 if line.strip() in ["STRUCTURE", "END_STRUCTURE"]:
374 continue
376 if line.strip().startswith(tuple("123456789")):
377 # Main section — require a dot-delimited name (e.g. "1. Intro").
378 parts = line.split(".", 1)
379 if len(parts) < 2 or not parts[1].strip():
380 continue
381 section_name = parts[1].strip()
382 current_section = {"name": section_name, "subsections": []}
383 structure.append(current_section)
384 elif line.strip().startswith("-") and current_section:
385 # Subsection with or without purpose
386 parts = line.strip("- ").split(
387 "|", 1
388 ) # Only split on first pipe
389 if len(parts) == 2:
390 current_section["subsections"].append(
391 {"name": parts[0].strip(), "purpose": parts[1].strip()}
392 )
393 elif len(parts) == 1 and parts[0].strip(): 393 ↛ 372line 393 didn't jump to line 372 because the condition on line 393 was always true
394 # Subsection without purpose - add default
395 current_section["subsections"].append(
396 {
397 "name": parts[0].strip(),
398 "purpose": f"Provide detailed information about {parts[0].strip()}",
399 }
400 )
402 # Check if the last section is source-related and remove it
403 if structure:
404 last_section = structure[-1]
405 section_name_lower = last_section["name"].lower()
406 source_keywords = [
407 "source",
408 "citation",
409 "reference",
410 "bibliography",
411 ]
413 # Only check the last section for source-related content
414 if any(
415 keyword in section_name_lower for keyword in source_keywords
416 ):
417 logger.info(
418 f"Removed source-related last section: {last_section['name']}"
419 )
420 structure = structure[:-1]
422 return structure
424 def _truncate_at_sentence_boundary(self, text: str, max_chars: int) -> str:
425 """Truncate text at a sentence boundary to preserve readability.
427 Attempts to cut at the last sentence-ending punctuation (.!?) before
428 the limit. If no suitable boundary is found within 80% of the limit,
429 falls back to hard truncation.
431 Args:
432 text: Text to truncate.
433 max_chars: Maximum characters allowed.
435 Returns:
436 Truncated text with ``[...truncated]`` marker if truncation
437 occurred, otherwise the original text unchanged.
438 """
439 if len(text) <= max_chars:
440 return text
442 truncated = text[:max_chars]
444 # Look for sentence boundaries (. ! ?) followed by space or newline
445 # Search backwards from the end for the last complete sentence
446 last_sentence_end = -1
447 for i in range(len(truncated) - 1, -1, -1):
448 if truncated[i] in ".!?" and (
449 i + 1 >= len(truncated) or truncated[i + 1] in " \n"
450 ):
451 last_sentence_end = i + 1
452 break
454 # Only use sentence boundary if it preserves at least 80% of content
455 min_acceptable = int(max_chars * 0.8)
456 if last_sentence_end > min_acceptable:
457 return truncated[:last_sentence_end] + "\n[...truncated]"
459 # Fall back to hard truncation
460 return truncated + "\n[...truncated]"
462 @staticmethod
463 def _normalize_heading_text(text: str) -> str:
464 """Normalize heading text for fuzzy comparison against subsection names.
466 Strips markdown bold markers, leading Roman-numeral / alphabetic /
467 numeric enumeration prefixes (``II.``, ``A.``, ``1.``), collapses
468 whitespace, and lowercases. Used only for matching — never written
469 back into the report.
470 """
471 cleaned = text.strip()
472 # Drop surrounding bold/italic markers the LLM wraps titles in
473 # (``**Title**``, ``*Title*``, ``__Title__``). Avoid regex here —
474 # CodeQL flags ``^[*_]+|[*_]+$`` as polynomial on long ``*`` runs.
475 cleaned = cleaned.strip("*_").strip()
476 cleaned = re.sub(
477 r"^(?:"
478 r"[IVXLCDM]+\.|" # Roman numerals: II. III. IV. (uppercase only to avoid "Mix." false strip)
479 r"[A-Z]\.|" # Single letters: A. B.
480 r"\d+\." # Arabic numerals: 1. 2.
481 r")\s+",
482 "",
483 cleaned,
484 )
485 # Normalise curly/smart quotes and dashes so "Author's" (U+2019)
486 # matches "Author's" (ASCII apostrophe) and en/em dashes collapse
487 # to a plain hyphen for comparison.
488 cleaned = cleaned.translate(
489 str.maketrans(
490 "\u2018\u2019\u201c\u201d\u2013\u2014\u2212", "''\"\"---"
491 )
492 )
493 return re.sub(r"\s+", " ", cleaned).strip().lower()
495 @classmethod
496 def _heading_restates_name(cls, heading_text: str, name: str) -> bool:
497 """Return True if *heading_text* is a restatement of *name*.
499 Handles exact matches, prefix matches (heading extends the name
500 with a colon/dash subtitle), and the reverse (name is a longer
501 form of a short heading). Empty names never match.
502 """
503 if not name or not heading_text:
504 return False
505 norm_heading = cls._normalize_heading_text(heading_text)
506 norm_name = cls._normalize_heading_text(name)
507 if not norm_heading or not norm_name: 507 ↛ 508line 507 didn't jump to line 508 because the condition on line 507 was never true
508 return False
509 if norm_heading == norm_name:
510 return True
511 # Heading extends the subsection name with a subtitle (e.g. "Name: subtitle")
512 # Delimiters only: bare space prefixes (e.g. "Introduction to …")
513 # are intentional non-matches. En/em dashes are already folded to
514 # ASCII "-" by _normalize_heading_text.
515 if norm_heading.startswith(norm_name):
516 rest = norm_heading[len(norm_name) :].lstrip()
517 if rest and rest[0] in (":", "-", "("):
518 return True
519 # Name extends a short heading with a subtitle.
520 if norm_name.startswith(norm_heading): 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true
521 rest = norm_name[len(norm_heading) :].lstrip()
522 if rest and rest[0] in (":", "-", "("):
523 return True
524 return False
526 def _strip_leading_heading(
527 self,
528 content: str,
529 subsection_name: str,
530 sibling_subsection_names: Optional[List[str]] = None,
531 ) -> str:
532 """Drop a single leading heading line only when it restates subsection or sibling name."""
533 leading = _LEADING_HEADING_RE.match(content)
534 if not leading:
535 return content
537 fence_spans = _get_code_fence_spans(content)
538 if _is_in_spans(leading.start(), fence_spans): 538 ↛ 539line 538 didn't jump to line 539 because the condition on line 538 was never true
539 return content
541 heading_text = re.sub(r"^\s*#{1,6}[ \t]+", "", leading.group(0)).strip()
542 # Subsection name: allow subtitle extensions (Name: subtitle)
543 if self._heading_restates_name(heading_text, subsection_name):
544 end_pos = leading.end()
545 while end_pos < len(content) and content[end_pos] in "\r\n":
546 end_pos += 1
547 return content[end_pos:]
548 # Siblings: require exact normalized equality (avoid over-strip of
549 # "### <Sibling>: Subtitle" organizers)
550 norm_heading = self._normalize_heading_text(heading_text)
551 for sibling in sibling_subsection_names or []:
552 if sibling and norm_heading == self._normalize_heading_text( 552 ↛ 551line 552 didn't jump to line 551 because the condition on line 552 was always true
553 sibling
554 ):
555 end_pos = leading.end()
556 while end_pos < len(content) and content[end_pos] in "\r\n":
557 end_pos += 1
558 return content[end_pos:]
559 return content
561 def _strip_leading_italic_purpose(
562 self,
563 content: str,
564 purpose: Optional[str] = None,
565 ) -> str:
566 """Drop a single leading italic purpose statement if it mirrors purpose or boilerplate."""
567 leading_italic = _LEADING_ITALIC_PURPOSE_RE.match(content)
568 if not leading_italic:
569 return content
571 fence_spans = _get_code_fence_spans(content)
572 if _is_in_spans(leading_italic.start(), fence_spans): 572 ↛ 573line 572 didn't jump to line 573 because the condition on line 572 was never true
573 return content
575 italic_raw = leading_italic.group(0).strip()
576 italic_text = italic_raw.strip("_*").strip()
577 should_strip = False
579 if purpose:
580 norm_italic = self._normalize_heading_text(italic_text)
581 norm_purpose = self._normalize_heading_text(purpose)
582 # Guard against degenerate purpose that normalizes to "" (e.g. "***")
583 # — startswith("") is always True and would strip any italic line.
584 if norm_italic and norm_purpose: 584 ↛ 593line 584 didn't jump to line 593 because the condition on line 584 was always true
585 if (
586 norm_italic == norm_purpose
587 or norm_italic.startswith(norm_purpose)
588 or norm_purpose.startswith(norm_italic)
589 or self._heading_restates_name(italic_text, purpose)
590 ):
591 should_strip = True
593 if not should_strip:
594 norm_italic = self._normalize_heading_text(italic_text)
595 if norm_italic: 595 ↛ 687line 595 didn't jump to line 687 because the condition on line 595 was always true
596 # Narrow unconditional boilerplate markers — always safe to strip
597 if norm_italic.startswith(
598 (
599 "purpose:",
600 "scope:",
601 "this subsection ",
602 "this section ",
603 )
604 ):
605 should_strip = True
606 # Verb-prefix boilerplate: check for relevance to purpose to avoid
607 # eating legitimate epigraphs like "_To review the evidence is to understand the past._"
608 # when purpose is unrelated (S2).
609 elif purpose is not None and norm_italic.startswith(
610 (
611 "to summarize",
612 "to summarise",
613 "to clarify",
614 "to examine",
615 "to outline",
616 "to provide",
617 "to describe",
618 "to detail",
619 "to analyze",
620 "to analyse",
621 "to explore",
622 "to present",
623 "to discuss",
624 "to evaluate",
625 "to investigate",
626 "to review",
627 "to assess",
628 "aims to",
629 "designed to",
630 )
631 ):
632 norm_purpose = self._normalize_heading_text(purpose)
633 if norm_purpose: 633 ↛ 687line 633 didn't jump to line 687 because the condition on line 633 was always true
634 _stop_words = {
635 "to",
636 "the",
637 "and",
638 "for",
639 "that",
640 "this",
641 "with",
642 "from",
643 "summarize",
644 "summarise",
645 "clarify",
646 "examine",
647 "outline",
648 "provide",
649 "describe",
650 "detail",
651 "analyze",
652 "analyse",
653 "explore",
654 "present",
655 "discuss",
656 "evaluate",
657 "investigate",
658 "review",
659 "assess",
660 "aims",
661 "designed",
662 "section",
663 "subsection",
664 }
665 purpose_words = {
666 w
667 for w in re.findall(
668 r"\b[a-z0-9]{3,}\b", norm_purpose.lower()
669 )
670 if w not in _stop_words
671 }
672 italic_words = {
673 w
674 for w in re.findall(
675 r"\b[a-z0-9]{3,}\b", norm_italic.lower()
676 )
677 if w not in _stop_words
678 }
679 if ( 679 ↛ 685line 679 didn't jump to line 685 because the condition on line 679 was never true
680 self._heading_restates_name(italic_text, purpose)
681 or norm_italic.startswith(norm_purpose)
682 or norm_purpose.startswith(norm_italic)
683 or bool(purpose_words & italic_words)
684 ):
685 should_strip = True
687 if should_strip:
688 end_pos = leading_italic.end()
689 while end_pos < len(content) and content[end_pos] in "\r\n":
690 end_pos += 1
691 return content[end_pos:]
692 return content
694 def _strip_embedded_bibliographies(self, content: str) -> str:
695 """Remove every embedded bibliography block outside code blocks.
697 Only removes a heading that looks like a bibliography *and* whose
698 following block looks like a citation list — this prevents
699 substantive headings such as "### Sources of Bias..." or
700 "### Sources. Data Collection Methodology" from being deleted with
701 their analysis body (R1). Consecutive bibliography headings no longer
702 consume intervening prose (R8).
703 """
704 fence_spans = _get_code_fence_spans(content)
705 pieces: List[str] = []
706 cursor = 0
707 bib_removed = False
709 for match in _BIBLIOGRAPHY_HEADING_RE.finditer(content):
710 if _is_in_spans(match.start(), fence_spans):
711 continue
713 after_heading = match.end()
714 next_heading_start = None
715 next_heading_is_bib = False
716 for nh in _NEXT_HEADING_RE.finditer(content, after_heading):
717 if not _is_in_spans(nh.start(), fence_spans): 717 ↛ 716line 717 didn't jump to line 716 because the condition on line 717 was always true
718 next_heading_start = nh.start()
719 # Peek if the next heading itself is a bibliography heading
720 # — if so, we will trim the current block to its citation
721 # list end rather than to the heading (R8).
722 heading_line = content[nh.start() :].split("\n", 1)[0]
723 if _BIBLIOGRAPHY_HEADING_RE.match(heading_line): 723 ↛ 724line 723 didn't jump to line 724 because the condition on line 723 was never true
724 next_heading_is_bib = True
725 break
727 block_end = (
728 next_heading_start
729 if next_heading_start is not None
730 else len(content)
731 )
732 block_text = content[after_heading:block_end]
734 # Body-shape safeguard: only treat as bibliography if the block
735 # contains citation-like lines. Require at least one citation
736 # marker, or >30% of non-empty lines look like citations, to
737 # avoid deleting substantive analysis headings that happen to
738 # match the label grammar (e.g. "### Sources for the Analysis"
739 # with prose body).
740 non_empty_lines = [
741 ln for ln in block_text.splitlines() if ln.strip()
742 ]
743 if not non_empty_lines: 743 ↛ 745line 743 didn't jump to line 745 because the condition on line 743 was never true
744 # Empty block — treat as bibliography (heading with no body)
745 is_bib_block = True
746 else:
747 citation_lines = sum(
748 1 for ln in non_empty_lines if _CITATION_LINE_RE.match(ln)
749 )
750 # Require explicit italic note boilerplate (e.g. "*(Note: ... bibliography ...)*")
751 # rather than matching arbitrary prose containing "bibliography" (S1).
752 has_bib_note = any(
753 _BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
754 )
755 if citation_lines == 0 and not has_bib_note:
756 # No citation shape and no italic bib note — preserve heading + body
757 continue
758 # If citation lines present, require they are not a tiny minority
759 # unless block is short (e.g. "- item" single line)
760 if citation_lines == 0:
761 is_bib_block = has_bib_note
762 elif len(non_empty_lines) <= 3:
763 is_bib_block = citation_lines >= 1
764 else:
765 is_bib_block = (
766 citation_lines / len(non_empty_lines)
767 ) >= 0.3 or citation_lines >= 2
769 if not is_bib_block: 769 ↛ 770line 769 didn't jump to line 770 because the condition on line 769 was never true
770 continue
772 # Passed safeguard — this is a real bibliography block to remove
773 bib_removed = True
774 block_start = match.start()
775 prefix = content[cursor:block_start]
776 hr_matches = list(_HR_LINE_RE.finditer(prefix))
777 if hr_matches:
778 cut_idx = len(prefix)
779 for hr_m in reversed(hr_matches):
780 between = prefix[hr_m.end() : cut_idx]
781 if between.strip():
782 break
783 cut_idx = hr_m.start()
784 if cut_idx < len(prefix): 784 ↛ 787line 784 didn't jump to line 787 because the condition on line 784 was always true
785 block_start = cursor + cut_idx
787 pieces.append(content[cursor:block_start])
789 # R8: consecutive bib headings — end at citation list, not at next bib heading
790 if next_heading_is_bib: 790 ↛ 792line 790 didn't jump to line 792 because the condition on line 790 was never true
791 # Find end of citation list within block_text
792 lines = block_text.splitlines(keepends=True)
793 offset = 0
794 last_citation_end = 0
795 for ln in lines:
796 if _CITATION_LINE_RE.match(ln) or (
797 _BIB_NOTE_RE.match(ln) and offset < 300
798 ):
799 last_citation_end = offset + len(ln)
800 # Also keep consecutive citation lines; stop at first
801 # non-citation prose that is not a blank/bib note?
802 offset += len(ln)
803 # Also include trailing blank lines after citations, but not
804 # intervening prose before next bib heading
805 if last_citation_end > 0:
806 # Advance cursor to after the citation list (preserve
807 # intervening prose between two bib headings)
808 cursor = after_heading + last_citation_end
809 # Skip following blank lines / HRs but not prose
810 while cursor < block_end and content[cursor] in " \t\r\n-":
811 # Only skip whitespace/HRs, stop at prose
812 # Peek next non-whitespace chunk
813 nxt = content[cursor:].lstrip(" \t\r\n")
814 if nxt.startswith("-") and nxt[1:2] in " \t":
815 # Another HR-like line, skip it
816 cursor += len(content[cursor:]) - len(nxt)
817 # consume the HR line
818 eol = content.find("\n", cursor)
819 cursor = eol + 1 if eol != -1 else len(content)
820 else:
821 break
822 if cursor >= block_end:
823 break
824 # Ensure we don't overshoot into next bib heading's prefix;
825 # if cursor is before next_heading_start, keep it, else
826 # fall back to next_heading_start
827 if cursor > block_end:
828 cursor = block_end
829 else:
830 cursor = block_end
831 else:
832 # End the removal at the citation list rather than at the end
833 # of the block, so a table, list or paragraph the model wrote
834 # after its bibliography survives the strip.
835 offset = 0
836 citation_end = 0
837 non_empty_seen = 0
838 trailing_content = False
839 for ln in block_text.splitlines(keepends=True):
840 substantive = bool(ln.strip())
841 non_empty_seen += substantive
842 # The note window is the classifier's first-three-non-empty-
843 # lines rule, so a block accepted as a bibliography always
844 # has a qualifying line here and cannot fall through.
845 is_bib_line = bool(
846 _CITATION_LINE_RE.match(ln)
847 or (_BIB_NOTE_RE.match(ln) and non_empty_seen <= 3)
848 or _HR_LINE_RE.match(ln)
849 )
850 if citation_end and substantive and not is_bib_line:
851 trailing_content = True
852 break
853 offset += len(ln)
854 if substantive and is_bib_line:
855 citation_end = offset
856 if not trailing_content:
857 # Nothing substantive follows, so the whole block is the
858 # bibliography and is removed as before.
859 citation_end = len(block_text)
861 cursor = after_heading + citation_end
863 if bib_removed:
864 pieces.append(content[cursor:])
865 return "".join(pieces)
866 return content
868 def _strip_subsection_boilerplate(
869 self,
870 content: str,
871 subsection_name: str,
872 section_name: str,
873 sibling_subsection_names: Optional[List[str]] = None,
874 purpose: Optional[str] = None,
875 ) -> str:
876 """Strip boilerplate the LLM tends to emit around subsection content.
878 Small and/or quantized local models routinely produce three artefacts
879 that pollute the rendered report even when the OUTPUT FORMAT RULES
880 spelled out in the subsection prompt explicitly forbid them:
882 1. A redundant leading heading (stripped by :meth:`_strip_leading_heading`)
883 2. An italic purpose statement (stripped by :meth:`_strip_leading_italic_purpose`)
884 3. An embedded bibliography block (stripped by :meth:`_strip_embedded_bibliographies`)
886 This helper normalises the per-subsection content so the rendered
887 report stays clean even when the model ignored the prompt. It is a
888 defensive cleanup, not a substitute for the prompt rules.
890 Args:
891 content: Raw ``current_knowledge`` returned for one subsection.
892 subsection_name: The subsection's name (used for matching and
893 logging).
894 section_name: The parent section's name (used for logging).
895 sibling_subsection_names: Optional names of other subsections
896 in the same section. When provided, a leading heading that
897 restates a *sibling's* name (context-bleed) is also stripped.
898 purpose: Optional purpose string for the subsection. Used to
899 verify if a leading italic line mirrors the subsection purpose.
901 Returns:
902 ``content`` with a redundant leading heading removed when it
903 restates this or a sibling subsection name, a leading
904 italic-purpose statement removed, and any embedded bibliography
905 blocks removed. Returns content unchanged if no artefacts are
906 found, aside from collapsing 3+ blank lines to 2 and
907 normalising trailing whitespace (collapsed outside code fences
908 only, so fenced blocks are never rewritten).
909 """
910 if not content:
911 return content
913 original_len = len(content)
914 new_content = self._strip_leading_heading(
915 content, subsection_name, sibling_subsection_names
916 )
917 new_content = self._strip_leading_italic_purpose(new_content, purpose)
918 new_content = self._strip_embedded_bibliographies(new_content)
920 # R2: fence-aware whitespace collapse — never rewrite inside code fences
921 fence_spans = _get_code_fence_spans(new_content)
922 if not fence_spans:
923 collapsed = re.sub(r"\n{3,}", "\n\n", new_content).strip()
924 else:
925 parts: List[str] = []
926 last = 0
927 for s, e in sorted(fence_spans):
928 # collapse outside fence
929 parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:s]))
930 # keep fence content verbatim
931 parts.append(new_content[s:e])
932 last = e
933 parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:]))
934 collapsed = "".join(parts).strip()
935 new_content = collapsed
936 if new_content: 936 ↛ 939line 936 didn't jump to line 939 because the condition on line 936 was always true
937 new_content += "\n"
939 if len(new_content) != original_len:
940 # R5: make silent deletions observable
941 if not new_content.strip() and content.strip(): 941 ↛ 942line 941 didn't jump to line 942 because the condition on line 941 was never true
942 logger.warning(
943 "Stripped subsection boilerplate emptied non-trivial content for "
944 f"'{section_name} > {subsection_name}': "
945 f"{original_len} -> {len(new_content)} chars — "
946 "check for over-strip (R1/R3 regression)"
947 )
948 elif len(new_content) < 0.5 * original_len:
949 logger.warning(
950 "Stripped subsection boilerplate heavily truncated content for "
951 f"'{section_name} > {subsection_name}': "
952 f"{original_len} -> {len(new_content)} chars"
953 )
954 else:
955 logger.debug(
956 "Stripped subsection boilerplate for "
957 f"'{section_name} > {subsection_name}': "
958 f"{original_len} -> {len(new_content)} chars"
959 )
960 return new_content
962 def _build_previous_context(self, accumulated_findings: List[str]) -> str:
963 """Build context block from previously generated sections.
965 Creates a formatted context block containing content from the last
966 N sections (defined by self.max_context_sections) with explicit instructions
967 not to repeat this content. Context is truncated if it exceeds
968 self.max_context_chars to stay safe for smaller local models.
970 Args:
971 accumulated_findings: List of previously generated section content,
972 each formatted as "[Section > Subsection]\\n{content}"
974 Returns:
975 Formatted context block with delimiters, or empty string if no
976 previous findings exist
977 """
978 if not accumulated_findings:
979 return ""
981 recent_findings = accumulated_findings[-self.max_context_sections :]
982 previous_context = "\n\n---\n\n".join(recent_findings)
984 # Truncate at sentence boundary if too long
985 if len(previous_context) > self.max_context_chars:
986 previous_context = self._truncate_at_sentence_boundary(
987 previous_context, self.max_context_chars
988 )
990 return (
991 f"\n\n=== CONTENT ALREADY WRITTEN (DO NOT REPEAT) ===\n"
992 f"{previous_context}\n"
993 f"=== END OF PREVIOUS CONTENT ===\n\n"
994 f"CRITICAL: The above content has already been written. Do NOT repeat "
995 f"these points, examples, or explanations. Focus on NEW information "
996 f"not covered above.\n"
997 )
999 def _research_and_generate_sections(
1000 self,
1001 initial_findings: Dict,
1002 structure: List[Dict],
1003 query: str,
1004 progress_callback=None,
1005 ) -> Dict[str, str]:
1006 """Research and generate content for each section in one step.
1008 This method processes sections sequentially, accumulating generated
1009 content as it goes. For each new section/subsection, it passes context
1010 from the last few previously generated sections to help the LLM avoid
1011 repetition.
1013 The context accumulation mechanism:
1014 - Tracks all generated content in accumulated_findings list
1015 - Before generating each section, builds context from recent findings
1016 - Uses self.max_context_sections (configurable, default: 3) to limit context size
1017 - Truncates context to self.max_context_chars (configurable, default: 4000) for safety
1018 - Includes explicit "DO NOT REPEAT" instructions with actual content
1020 Args:
1021 initial_findings: Results from initial research phase, may contain
1022 questions_by_iteration to preserve search continuity
1023 structure: List of section definitions, each with name and subsections
1024 query: Original user query for context
1026 Returns:
1027 Dict mapping section names to their generated markdown content
1028 """
1029 sections = {}
1031 # Accumulate content from previous sections to avoid repetition
1032 accumulated_findings: List[str] = []
1034 # Count total subsections for progress tracking
1035 total_subsections = sum(
1036 max(len(section.get("subsections", [])), 1) for section in structure
1037 )
1038 completed_subsections = 0
1040 # Preserve questions from initial research to avoid repetition
1041 # This follows the same pattern as citation tracking (all_links_of_system)
1042 existing_questions = initial_findings.get("questions_by_iteration", {})
1043 if existing_questions:
1044 # Set questions on both search system and its strategy
1045 if hasattr(self.search_system, "questions_by_iteration"): 1045 ↛ 1051line 1045 didn't jump to line 1051 because the condition on line 1045 was always true
1046 self.search_system.questions_by_iteration = (
1047 existing_questions.copy()
1048 )
1050 # More importantly, set it on the strategy which actually uses it
1051 if hasattr(self.search_system, "strategy") and hasattr( 1051 ↛ 1061line 1051 didn't jump to line 1061 because the condition on line 1051 was always true
1052 self.search_system.strategy, "questions_by_iteration"
1053 ):
1054 self.search_system.strategy.questions_by_iteration = (
1055 existing_questions.copy()
1056 )
1057 logger.info(
1058 f"Initialized strategy with {len(existing_questions)} iterations of previous questions"
1059 )
1061 for i, section in enumerate(structure, 1):
1062 logger.info(f"Processing section: {section['name']}")
1063 section_content = []
1065 section_content.append(f"# {i}. {section['name']}\n")
1067 # If section has no subsections, create one from the section itself
1068 if not section["subsections"]:
1069 # Parse section name for purpose
1070 if "|" in section["name"]:
1071 parts = section["name"].split("|", 1)
1072 section["subsections"] = [
1073 {"name": parts[0].strip(), "purpose": parts[1].strip()}
1074 ]
1075 else:
1076 # No purpose provided - use section name as subsection
1077 section["subsections"] = [
1078 {
1079 "name": section["name"],
1080 "purpose": f"Provide comprehensive content for {section['name']}",
1081 }
1082 ]
1084 # Process each subsection by directly researching it
1085 for j, subsection in enumerate(section["subsections"], 1):
1086 # Only add subsection header if there are multiple subsections
1087 if len(section["subsections"]) > 1:
1088 section_content.append(f"## {i}.{j} {subsection['name']}\n")
1089 section_content.append(f"_{subsection['purpose']}_\n\n")
1091 # Get other subsections in this section for context
1092 other_subsections = [
1093 f"- {s['name']}: {s['purpose']}"
1094 for s in section["subsections"]
1095 if s["name"] != subsection["name"]
1096 ]
1097 other_subsections_text = (
1098 "\n".join(other_subsections)
1099 if other_subsections
1100 else "None"
1101 )
1103 # Get all other sections for broader context
1104 other_sections = [
1105 f"- {s['name']}"
1106 for s in structure
1107 if s["name"] != section["name"]
1108 ]
1109 other_sections_text = (
1110 "\n".join(other_sections) if other_sections else "None"
1111 )
1113 # Check if this is actually a section-level content (only one subsection, likely auto-created)
1114 is_section_level = len(section["subsections"]) == 1
1116 # Build context from previously generated sections to avoid repetition
1117 previous_context_section = self._build_previous_context(
1118 accumulated_findings
1119 )
1121 # Generate appropriate search query
1122 if is_section_level:
1123 # Section-level prompt - more comprehensive
1124 subsection_query = (
1125 f"Research task: Create comprehensive content for the '{subsection['name']}' section in a report about '{query}'. "
1126 f"Section purpose: {subsection['purpose']} "
1127 f"\n"
1128 f"Other sections in the report:\n{other_sections_text}\n"
1129 f"{previous_context_section}"
1130 f"This is a standalone section requiring comprehensive coverage of its topic. "
1131 f"Provide a thorough exploration that may include synthesis of information from previous sections where relevant. "
1132 f"Include unique insights, specific examples, and concrete data. "
1133 f"Use tables to organize information where applicable. "
1134 f"For conclusion sections: synthesize key findings and provide forward-looking insights. "
1135 f"Build upon the research findings from earlier sections to create a cohesive narrative."
1136 f"{_SUBSECTION_OUTPUT_GUIDANCE}"
1137 )
1138 else:
1139 # Subsection-level prompt - more focused
1140 subsection_query = (
1141 f"Research task: Create content for subsection '{subsection['name']}' in a report about '{query}'. "
1142 f"This subsection's purpose: {subsection['purpose']} "
1143 f"Part of section: '{section['name']}' "
1144 f"\n"
1145 f"Other sections in the report:\n{other_sections_text}\n"
1146 f"\n"
1147 f"Other subsections in this section will cover:\n{other_subsections_text}\n"
1148 f"{previous_context_section}"
1149 f"Focus ONLY on information specific to your subsection's purpose. "
1150 f"Include unique details, specific examples, and concrete data. "
1151 f"Use tables to organize information where applicable. "
1152 f"IMPORTANT: Avoid repeating information that would logically be covered in other sections - focus on what makes this subsection unique. "
1153 f"Previous research exists - find specific angles for this subsection."
1154 f"{_SUBSECTION_OUTPUT_GUIDANCE}"
1155 )
1157 logger.info(
1158 f"Researching subsection: {subsection['name']} with query: {subsection_query}"
1159 )
1161 # Report progress and check for cancellation
1162 if progress_callback:
1163 pct = int(
1164 10
1165 + (completed_subsections / max(total_subsections, 1))
1166 * 80
1167 )
1168 progress_callback(
1169 f"Researching: {section['name']} > {subsection['name']}",
1170 pct,
1171 {
1172 "phase": "report_section_research",
1173 "subsection": subsection["name"],
1174 },
1175 )
1177 # Fix iteration override: modify strategy's settings_snapshot
1178 # which is read dynamically via get_setting()
1179 strategy = self.search_system.strategy
1180 original_iterations = strategy.settings_snapshot.get(
1181 "search.iterations"
1182 )
1183 had_iterations_key = (
1184 "search.iterations" in strategy.settings_snapshot
1185 )
1186 strategy.settings_snapshot["search.iterations"] = 1
1187 # Belt-and-suspenders: also override max_iterations for
1188 # strategies that cache it at __init__ time
1189 original_max_iter = getattr(strategy, "max_iterations", None)
1190 strategy.max_iterations = 1
1192 try:
1193 # Perform search for this subsection
1194 subsection_results = self.search_system.analyze_topic(
1195 subsection_query
1196 )
1197 finally:
1198 # Restore original iteration settings
1199 if had_iterations_key:
1200 strategy.settings_snapshot["search.iterations"] = (
1201 original_iterations
1202 )
1203 else:
1204 strategy.settings_snapshot.pop(
1205 "search.iterations", None
1206 )
1207 if original_max_iter is not None: 1207 ↛ 1210line 1207 didn't jump to line 1210 because the condition on line 1207 was always true
1208 strategy.max_iterations = original_max_iter
1210 completed_subsections += 1
1212 # Add the researched content for this subsection
1213 if subsection_results.get("current_knowledge"):
1214 generated_content = subsection_results["current_knowledge"]
1215 # Strip the boilerplate the LLM tends to emit around
1216 # subsections: a redundant leading heading that mirrors
1217 # the framework's own `## i.j Name` heading, an italic
1218 # purpose statement that mirrors the framework's
1219 # `_<purpose>_` subtitle, and an embedded
1220 # '## Sources' / '## References' bibliography block
1221 # (the framework appends one master '## Sources' to
1222 # the whole report). Small/quantized local models
1223 # routinely ignore the OUTPUT FORMAT RULES in the
1224 # prompt above; this normalisation step guarantees a
1225 # clean rendered output regardless.
1226 sibling_names = [
1227 s["name"]
1228 for s in section["subsections"]
1229 if s["name"] != subsection["name"]
1230 ]
1231 try:
1232 generated_content = self._strip_subsection_boilerplate(
1233 generated_content,
1234 subsection_name=subsection["name"],
1235 section_name=section["name"],
1236 sibling_subsection_names=sibling_names,
1237 purpose=subsection.get("purpose"),
1238 )
1239 except Exception:
1240 logger.exception(
1241 "Boilerplate strip failed for "
1242 f"'{section['name']} > {subsection['name']}' — "
1243 "using raw LLM content"
1244 )
1245 # Fall back to raw content so a cosmetic step never
1246 # aborts the entire multi-section report.
1247 generated_content = (
1248 subsection_results.get("current_knowledge", "")
1249 or ""
1250 )
1251 if generated_content.strip(): 1251 ↛ 1258line 1251 didn't jump to line 1258 because the condition on line 1251 was always true
1252 section_content.append(generated_content)
1253 # Accumulate for context in subsequent sections
1254 accumulated_findings.append(
1255 f"[{section['name']} > {subsection['name']}]\n{generated_content}"
1256 )
1257 else:
1258 section_content.append(
1259 "*Limited information was found for this subsection.*\n"
1260 )
1261 else:
1262 section_content.append(
1263 "*Limited information was found for this subsection.*\n"
1264 )
1266 section_content.append("\n\n")
1268 # Combine all content for this section
1269 sections[section["name"]] = "\n".join(section_content)
1271 return sections
1273 def _generate_sections(
1274 self,
1275 initial_findings: Dict,
1276 _section_research: Dict[str, List[Dict]],
1277 structure: List[Dict],
1278 query: str,
1279 ) -> Dict[str, str]:
1280 """
1281 This method is kept for compatibility but no longer used.
1282 The functionality has been moved to _research_and_generate_sections.
1283 """
1284 return {}
1286 def _format_final_report(
1287 self,
1288 sections: Dict[str, str],
1289 structure: List[Dict],
1290 query: str,
1291 ) -> Dict:
1292 """Format the final report with table of contents and sections."""
1293 # Generate TOC
1294 toc = ["# Table of Contents\n"]
1295 for i, section in enumerate(structure, 1):
1296 toc.append(f"{i}. **{section['name']}**")
1297 if len(section["subsections"]) > 1:
1298 for j, subsection in enumerate(section["subsections"], 1):
1299 toc.append(
1300 f" {i}.{j} {subsection['name']} | _{subsection['purpose']}_"
1301 )
1303 # Combine TOC and sections
1304 report_parts = ["\n".join(toc), ""]
1306 # Add a summary of the research
1307 report_parts.append("# Research Summary")
1308 report_parts.append(
1309 "This report was researched using an advanced search system."
1310 )
1311 report_parts.append(
1312 "Research included targeted searches for each section and subsection."
1313 )
1314 report_parts.append("\n---\n")
1316 # Add each section's content
1317 for section in structure:
1318 if section["name"] in sections:
1319 report_parts.append(sections[section["name"]])
1320 report_parts.append("")
1322 # Format links from search system
1323 # Get utilities module dynamically to avoid circular imports
1324 utilities = importlib.import_module("local_deep_research.utilities")
1325 # Imported here for the same reason, and directly rather than off
1326 # the module object above so the count is computed by the real
1327 # grouping helper.
1328 from .utilities.search_utilities import count_distinct_sources
1330 formatted_all_links = (
1331 utilities.search_utilities.format_links_to_markdown(
1332 all_links=self.search_system.all_links_of_system
1333 )
1334 )
1336 # Create final report with all parts. The Sources tail is
1337 # kept here so in-memory consumers (MCP `generate_report`,
1338 # programmatic API) get the full assembled blob unchanged.
1339 # The DB save site (research_service.py) strips this Sources
1340 # section via format_document_split before persisting, so the
1341 # answer-only invariant on report_content still holds.
1342 final_report_content = "\n\n".join(report_parts)
1343 # Explicit "\n\n" separator: downstream regex consumers
1344 # (_SOURCES_SECTION_PATTERNS in text_optimization/citation_formatter.py
1345 # and _LEGACY_SOURCES_RE in web/services/report_assembly_service.py)
1346 # use line-anchored `re.MULTILINE` matching. Today the trailing
1347 # newlines produced by `"\n\n".join` happen to keep `## Sources`
1348 # at the start of a line, but that is incidental; an explicit
1349 # separator preserves the invariant against future section
1350 # template changes.
1351 #
1352 # The HTML-comment sentinel around the appended `## Sources`
1353 # block lets ``format_document_split`` locate this section
1354 # unambiguously, even when the LLM has emitted its own
1355 # `## Sources` header earlier in the prose. The legacy regex
1356 # patterns would otherwise match the first `## Sources` they
1357 # find and could over-strip a multi-section report where the
1358 # LLM happened to include an inline sources block. Without the
1359 # sentinel, a 1380-source run legitimately pushes the
1360 # answer/sources ratio below the 50% safety threshold and the
1361 # splitter logs a misleading "over-stripped" warning.
1362 # The sentinel is imported at module top; no circular-import
1363 # concern at use time.
1364 final_report_content += (
1365 f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
1366 f"## Sources\n\n{formatted_all_links}"
1367 )
1369 # Create metadata dictionary
1370 metadata = {
1371 "generated_at": datetime.now(UTC).isoformat(),
1372 # SOURCES, not entries. ``all_links_of_system`` holds one
1373 # entry per distinct (url, snippet) pair under the LangGraph
1374 # strategy, and raw un-deduped engine dicts under the others,
1375 # so its length counts occurrences. This number is reported
1376 # as "sources", so it must group the way the ## Sources block
1377 # above does.
1378 "initial_sources": count_distinct_sources(
1379 self.search_system.all_links_of_system
1380 ),
1381 "sections_researched": len(structure),
1382 "searches_per_section": self.searches_per_section,
1383 "query": query,
1384 }
1386 # Return both content and metadata
1387 return {"content": final_report_content, "metadata": metadata}
1389 def _generate_error_report(self, query: str, error_msg: str) -> str:
1390 return f"=== ERROR REPORT ===\nQuery: {query}\nError: {error_msg}"