Coverage for src/local_deep_research/report_generator.py: 88%
612 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1import importlib
2import re
3from typing import Any, Dict, List, Optional
4from datetime import datetime, UTC
6from langchain_core.language_models import BaseChatModel
7from loguru import logger
9# Fix circular import by importing directly from source modules
10from .config.llm_config import get_llm
11from .config.thread_settings import get_setting_from_snapshot
12from .search_system import AdvancedSearchSystem
13from .text_optimization.citation_formatter import (
14 LDR_APPENDED_SOURCES_SENTINEL,
15)
16from .utilities.json_utils import get_llm_response_text
17from .utilities.llm_utils import invoke_llm_sync
19# Default constants for context accumulation to avoid repetition
20# These are used as fallbacks when settings are not available
21DEFAULT_MAX_CONTEXT_SECTIONS = (
22 3 # Number of previous sections to include as context
23)
24DEFAULT_MAX_CONTEXT_CHARS = (
25 4000 # Max characters for context (safe for smaller local models)
26)
28# Spelled-out output rules appended to every per-subsection prompt. Small
29# and/or quantized local models routinely ignore long, vague instructions,
30# so this is intentionally short, numbered, and refers to the actual
31# rendering consequences (a duplicate heading, a duplicate bibliography)
32# rather than abstract guidance.
33#
34# The framework always (a) inserts `## i.j Name` itself before appending
35# LLM content and (b) appends one consolidated `## Sources` block to the
36# whole report at the end of _format_final_report. The LLM does not need
37# to repeat either of those things, and historically has — producing
38# visibly stacked duplicate headings and duplicate "## Sources" sections
39# per subsection.
40_SUBSECTION_OUTPUT_GUIDANCE = (
41 "\n\nOUTPUT FORMAT RULES (FOLLOW EXACTLY):\n"
42 "1. Do NOT start your output with a Markdown heading of any level "
43 "(#, ##, ###, ####, etc.). The framework already inserts the "
44 "subsection heading for you; starting with your own heading line "
45 "creates a visible duplicate next to it in the final report.\n"
46 "2. Do NOT end your output with a sources, references, bibliography, "
47 "citations, key references, or selected bibliography section or list. "
48 "Do NOT include any trailing heading or label (such as 'Sources:', "
49 "'**Sources:**', or 'References:'), and do NOT append a list or block "
50 "of citations/links at the end of your response. Citations must ONLY "
51 "appear inline within sentences (e.g. 'According to [1], ...'). "
52 "The framework automatically compiles and appends the consolidated "
53 "source list after all subsections are completed; including your own "
54 "trailing sources or bibliography duplicates it.\n"
55 "3. Begin your output with prose (a paragraph or a table), not a "
56 "heading and not an italic purpose statement. You may use '###' / "
57 "'####' / deeper levels for internal sub-subheadings inside this "
58 "subsection only."
59)
61# Pre-compiled normalisers used by _strip_subsection_boilerplate. These
62# are best-effort cleanups that run on the LLM's raw `current_knowledge`
63# before it is appended to the section — they do NOT touch the
64# framework's own headings or its trailing '## Sources' block.
65#
66# Leading heading: any ATX heading (level 1-6) at the very start of the
67# subsection content. Limited to a single match so we never strip a real
68# sub-subheading used to organise the body. Real reports show the LLM
69# almost always opens with a redundant ``### <subsection name>`` (or a
70# sibling's name, or a Roman-numeral-prefixed restatement) immediately
71# under the framework's own ``## i.j Name`` heading.
72_LEADING_HEADING_RE = re.compile(
73 r"\A[ \t\n]*[ \t]{0,3}#{1,6}[ \t]+[^\n]*",
74)
76# Leading italic purpose statement that mirrors the framework's
77# ``_<purpose>_`` subtitle (e.g. ``_To summarise the section's scope..._``).
78# Supports single underscores (_..._) or single asterisks (*...*).
79# Limited to a single match at the start.
80_LEADING_ITALIC_PURPOSE_RE = re.compile(
81 r"\A[ \t\n]*(?:_[^_\n]+_|(?:\*(?!\*)[^*\n]+\*))[ \t]*",
82)
84_BIB_KEYWORD_PATTERN = (
85 r"(?:"
86 r"Sources?|References?|Bibliograph(?:y|ies)|Citations?|"
87 r"Key[ \t]+References?|Selected[ \t]+Bibliograph(?:y|ies)|"
88 r"Works?[ \t]+Cited|Cited[ \t]+Works?|"
89 r"Additional[ \t]+Resources?|Further[ \t]+Reading"
90 r")"
91)
93_BIB_HEADINGS_CHAIN = (
94 rf"{_BIB_KEYWORD_PATTERN}"
95 rf"(?:[ \t]*(?:,|and|&|/)[ \t]+{_BIB_KEYWORD_PATTERN})*"
96)
98# ATX-style heading (with #) allowing:
99# - levels 1-6 with optional empty heading line preceding it (e.g. "###\n**Sources:**")
100# - optional section numbers (e.g. "1." or "1.1")
101# - optional bold/italic markers ("**Sources**", "*References:*")
102# - optional trailing colons, dashes, periods, or qualifiers
103_ATX_BIB_HEADING = (
104 rf"[ \t]{{0,3}}#{{1,6}}[ \t]*(?:\n[ \t]*)?"
105 rf"(?:\d+(?:\.\d+)*\.?[ \t]+)?"
106 rf"(?:\*\*|\*|__|_)?{_BIB_HEADINGS_CHAIN}(?:\*\*|\*|__|_)?(?:"
107 rf"[ \t]*[:\-\u2013\u2014.][ \t]*[^\n]*"
108 rf"|[ \t]+for[ \t]+[^\n]+"
109 rf"|[ \t]*\([^\n]+\)"
110 rf")?[ \t]*$"
111)
113# Pseudo-heading (without #) where the keyword line acts as a section header:
114# - bold or italic wrapped: "**Sources:**", "**Sources**", "*References:*"
115# - or plain keyword ending with a colon or dash: "Sources:", "References:"
116_PSEUDO_BIB_HEADING = (
117 r"[ \t]{0,3}"
118 rf"(?:"
119 rf"(?:\*\*|\*|__|_){_BIB_HEADINGS_CHAIN}[:\-\u2013\u2014.]?(?:\*\*|\*|__|_)"
120 rf"|"
121 rf"{_BIB_HEADINGS_CHAIN}[ \t]*[:\-\u2013\u2014]"
122 rf")[ \t]*$"
123)
125# Bibliography-style heading at levels 1-6 or pseudo-heading line.
126_BIBLIOGRAPHY_HEADING_RE = re.compile(
127 rf"(?m)^(?:{_ATX_BIB_HEADING}|{_PSEUDO_BIB_HEADING})",
128 re.IGNORECASE,
129)
131# Next ATX heading at levels 1-6 — marks the end of a bibliography block
132# when more subsection content follows it (common when the LLM appends a
133# "Selected Bibliography" mid-stream and then continues writing).
134_NEXT_HEADING_RE = re.compile(r"(?m)^[ \t]{0,3}#{1,6}[ \t]+")
136# Decorative horizontal rules the LLM often wraps around bibliography
137# blocks (``---``, ``***``, or ``___`` on their own line).
138_HR_LINE_RE = re.compile(
139 r"(?m)^[ \t]{0,3}(?:(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})[ \t]*\n?"
140)
142# Empty heading line (e.g. `###`) or decorative horizontal rule line (`---`, `***`, `___`)
143_EMPTY_HEADING_OR_HR_RE = re.compile(
144 r"^[ \t]*(?:#{1,6}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})[ \t]*$"
145)
147# Reference-style link definitions in markdown: [1]: https://... or [name]: url
148_MD_LINK_DEF_RE = re.compile(r"^[ \t]*\[[^\]]+\]:[ \t]*\S+")
150# Ordinary ordered or bulleted list item prefix
151_ORDERED_OR_BULLET_LIST_RE = re.compile(r"^[ \t]*(?:[-*•]|\d+[\.\)])\s+")
153# Citation-list shape for ATX headings (## Sources, ### References):
154# digit+dot, bullet, bracket citation start (single or double brackets),
155# lenticular brackets, URL, DOI, or markdown link.
156# Excludes markdown reference-style link definitions ("[1]: https://...").
157_CITATION_LINE_RE = re.compile(
158 r"^\s*(?:"
159 r"\d+[\.\)]\s+"
160 r"|[-*•]\s+"
161 r"|\[\[\d+\]\](?:\([^)]*\))?"
162 r"|\[\d+\](?!\:)"
163 r"|【\d+】"
164 r"|\[cite\d*\]"
165 r"|\bhttps?://"
166 r"|\bdoi:\s*\S+"
167 r"|\[[^\]]+\]\((?:https?://|/library/)[^)]+\)"
168 r")",
169 re.IGNORECASE,
170)
172# Citation-list shape for pseudo-headings (Sources:, **Sources** without #):
173# Must carry explicit citation markers (brackets, links, URLs, DOIs), NOT plain numbered/bullet lists.
174_PSEUDO_BIB_CITATION_LINE_RE = re.compile(
175 r"^\s*(?:(?:[-*•]|\d+[\.\)])\s+)?"
176 r"(?:"
177 r"\[\[\d+\]\](?:\([^)]*\))?"
178 r"|\[\d+\](?!\:)"
179 r"|【\d+】"
180 r"|\[cite\d*\]"
181 r"|\bhttps?://"
182 r"|\bdoi:\s*\S+"
183 r"|\[[^\]]+\]\((?:https?://|/library/)[^)]+\)"
184 r")",
185 re.IGNORECASE,
186)
188# Single citation token for concatenated line check:
189# Supports single brackets [1], double brackets [[1]], optional markdown link target,
190# [cite1], or Chinese/Japanese lenticular brackets 【1】
191_CONCATENATED_CITATION_ITEM = (
192 r"(?:(?:\[\[\d+\]\]|\[\d+\]|\[cite\d*\])(?:\([^)]*\))?|【\d+】)"
193)
195# Line composed purely of citation markers, brackets, links, and commas.
196# Structured without overlapping leading/trailing whitespace quantifiers to prevent ReDoS.
197_CONCATENATED_CITATIONS_LINE_RE = re.compile(
198 rf"^(?:[,\s]*{_CONCATENATED_CITATION_ITEM})+[,\s]*$"
199)
201# Unheaded citation list item start (for trailing citation blocks lacking any heading).
202# Excludes numbered/bullet lists, markdown link definitions, and bare URLs without citation markers.
203# Requires double brackets [[...]], single brackets with link target/source, cite tag, or DOI.
204_UNHEADED_CITATION_ITEM_RE = re.compile(
205 r"^\s*(?:"
206 r"\[\[\d+\]\](?:\([^)]*\))?"
207 r"|\[\d+\](?!\:)(?:\((?:https?://|/library/)[^)]+\)|\s+(?:https?://|doi:))"
208 r"|\[cite\d*\]"
209 r"|doi:\s*\S+"
210 r")",
211 re.IGNORECASE,
212)
214# Note line matching LLM boilerplate (e.g. "*(Note: ... bibliography ...)*" or
215# "[Note: The framework will append...]")
216_BIB_NOTE_RE = re.compile(
217 r"^\s*(?:"
218 r"[\*_]\s*\(?Note\b[^\n]*?\b(?:bibliography|sources?|references?|citations?)\b"
219 r"|\[\s*Note\b"
220 r"(?=[^\n]*\b(?:consolidat\w*|append\w*|omit\w*|duplicat\w*|placeholder\w*)\b)"
221 r"[^\n]*?\b(?:bibliography|sources?|references?|citations?)\b)",
222 re.IGNORECASE,
223)
226def _get_code_fence_spans(text: str) -> List[tuple[int, int]]:
227 """Return character index ranges (start, end) that are inside code blocks.
229 Recognizes:
230 - Backtick fenced blocks (``` or ```` etc.), paired by delimiter type and length.
231 - Tilde fenced blocks (~~~ or ~~~~ etc.), paired by delimiter type and length.
232 - Indented code blocks (lines indented by 4+ spaces or tabs, preceded by a blank line).
233 """
234 spans: List[tuple[int, int]] = []
235 if not text:
236 return spans
238 lines = text.splitlines(keepends=True)
239 in_fence = False
240 fence_char = ""
241 fence_len = 0
242 fence_start = 0
244 in_indented = False
245 indented_start = 0
246 last_indented_end = 0
247 prev_blank = True
249 line_offset = 0
250 fence_open_re = re.compile(r"^[ \t]{0,3}(`{3,}|~{3,})")
252 for line in lines:
253 line_start = line_offset
254 line_end = line_offset + len(line)
255 line_offset = line_end
257 stripped = line.rstrip("\r\n")
259 if in_fence:
260 close_re = (
261 rf"^[ \t]{{0,3}}{re.escape(fence_char)}{{{fence_len},}}[ \t]*$"
262 )
263 if re.match(close_re, stripped):
264 in_fence = False
265 spans.append((fence_start, line_end))
266 continue
268 open_match = fence_open_re.match(stripped)
269 if open_match:
270 if in_indented: 270 ↛ 271line 270 didn't jump to line 271 because the condition on line 270 was never true
271 in_indented = False
272 spans.append((indented_start, last_indented_end))
274 in_fence = True
275 delim = open_match.group(1)
276 fence_char = delim[0]
277 fence_len = len(delim)
278 fence_start = line_start
279 prev_blank = False
280 continue
282 is_indented = stripped.startswith(" ") or stripped.startswith("\t")
283 is_blank = not stripped.strip()
285 if in_indented:
286 if is_indented:
287 last_indented_end = line_end
288 elif is_blank:
289 pass
290 else:
291 in_indented = False
292 spans.append((indented_start, last_indented_end))
293 else:
294 if is_indented and prev_blank:
295 in_indented = True
296 indented_start = line_start
297 last_indented_end = line_end
299 prev_blank = is_blank
301 if in_fence: 301 ↛ 302line 301 didn't jump to line 302 because the condition on line 301 was never true
302 spans.append((fence_start, len(text)))
303 elif in_indented: 303 ↛ 304line 303 didn't jump to line 304 because the condition on line 303 was never true
304 spans.append((indented_start, last_indented_end))
306 return spans
309def _is_in_spans(pos: int, spans: List[tuple[int, int]]) -> bool:
310 """Return True if pos falls within any range in spans."""
311 return any(start <= pos < end for start, end in spans)
314def get_report_generator(search_system=None):
315 """Return an instance of the report generator with default settings.
317 Args:
318 search_system: Optional existing AdvancedSearchSystem to use
319 """
320 return IntegratedReportGenerator(search_system=search_system)
323class IntegratedReportGenerator:
324 def __init__(
325 self,
326 searches_per_section: int = 2,
327 search_system=None,
328 llm: BaseChatModel | None = None,
329 settings_snapshot: Optional[Dict] = None,
330 ):
331 """
332 Args:
333 searches_per_section: Number of searches to perform for each
334 section in the report.
335 search_system: Custom search system to use, otherwise just uses
336 the default.
337 llm: Custom LLM to use. Required if search_system is not provided.
338 settings_snapshot: Optional settings snapshot for configurable values.
340 """
341 # If search_system is provided, use its LLM; otherwise use the provided LLM
342 self._owns_llm = False
343 if search_system:
344 self.search_system = search_system
345 self.model = llm or search_system.model
346 elif llm:
347 self.model = llm
348 self.search_system = AdvancedSearchSystem(llm=self.model) # type: ignore[call-arg]
349 else:
350 # Fallback for backwards compatibility - will only work with auth
351 self._owns_llm = True
352 self.model = get_llm()
353 self.search_system = AdvancedSearchSystem(llm=self.model) # type: ignore[call-arg]
355 self.searches_per_section = (
356 searches_per_section # Control search depth per section
357 )
359 # Load context settings from snapshot or use defaults
360 self.max_context_sections = get_setting_from_snapshot(
361 "report.max_context_sections",
362 default=DEFAULT_MAX_CONTEXT_SECTIONS,
363 settings_snapshot=settings_snapshot,
364 )
365 self.max_context_chars = get_setting_from_snapshot(
366 "report.max_context_chars",
367 default=DEFAULT_MAX_CONTEXT_CHARS,
368 settings_snapshot=settings_snapshot,
369 )
371 def close(self) -> None:
372 """Close the LLM client if this instance created it."""
373 from .utilities.resource_utils import safe_close
375 if self._owns_llm:
376 safe_close(self.model, "report generator LLM")
378 def generate_report(
379 self,
380 initial_findings: Dict,
381 query: str,
382 progress_callback=None,
383 ) -> Dict:
384 """Generate a complete research report with section-specific research.
386 Args:
387 initial_findings: Results from initial research phase.
388 query: Original user query.
389 progress_callback: Optional callable(message, progress_percent, metadata)
390 for reporting progress (0-100%) and checking cancellation.
391 """
393 # Step 1: Determine structure
394 if progress_callback:
395 progress_callback(
396 "Determining report structure",
397 0,
398 {"phase": "report_structure"},
399 )
400 structure = self._determine_report_structure(initial_findings, query)
402 # Step 2: Research and generate content for each section in one step
403 sections = self._research_and_generate_sections(
404 initial_findings,
405 structure,
406 query,
407 progress_callback=progress_callback,
408 )
410 # Step 3: Format final report
411 if progress_callback:
412 progress_callback(
413 "Formatting final report",
414 90,
415 {"phase": "report_formatting"},
416 )
417 report = self._format_final_report(sections, structure, query)
419 if progress_callback:
420 progress_callback(
421 "Report complete", 100, {"phase": "report_complete"}
422 )
424 return report
426 def _determine_report_structure(
427 self, findings: Dict, query: str
428 ) -> List[Dict]:
429 """Analyze content and determine the optimal detailed report structure."""
430 combined_content = findings["current_knowledge"]
431 prompt = f"""
432 Analyze this research content about: {query}
434 Content Summary:
435 {combined_content}
437 Determine the most appropriate detailed report structure by:
438 1. Analyzing the type of content (technical, business, academic, etc.)
439 2. Identifying main themes and logical groupings
440 3. Considering the depth and breadth of the research
442 Return a table of contents structure in this exact format:
443 STRUCTURE
444 1. [Section Name]
445 - [Subsection] | [purpose]
446 2. [Section Name]
447 - [Subsection] | [purpose]
448 ...
449 END_STRUCTURE
451 Make the structure specific to the content, not generic.
452 Each subsection must include its purpose after the | symbol.
453 DO NOT include sections about sources, citations, references, or methodology.
454 """
456 response = get_llm_response_text(invoke_llm_sync(self.model, prompt))
458 # Parse the structure
459 structure: List[Dict[str, Any]] = []
460 current_section: Optional[Dict[str, Any]] = None
462 for line in response.split("\n"):
463 if line.strip() in ["STRUCTURE", "END_STRUCTURE"]:
464 continue
466 if line.strip().startswith(tuple("123456789")):
467 # Main section — require a dot-delimited name (e.g. "1. Intro").
468 parts = line.split(".", 1)
469 if len(parts) < 2 or not parts[1].strip():
470 continue
471 section_name = parts[1].strip()
472 current_section = {"name": section_name, "subsections": []}
473 structure.append(current_section)
474 elif line.strip().startswith("-") and current_section:
475 # Subsection with or without purpose
476 parts = line.strip("- ").split(
477 "|", 1
478 ) # Only split on first pipe
479 if len(parts) == 2:
480 current_section["subsections"].append(
481 {"name": parts[0].strip(), "purpose": parts[1].strip()}
482 )
483 elif len(parts) == 1 and parts[0].strip(): 483 ↛ 462line 483 didn't jump to line 462 because the condition on line 483 was always true
484 # Subsection without purpose - add default
485 current_section["subsections"].append(
486 {
487 "name": parts[0].strip(),
488 "purpose": f"Provide detailed information about {parts[0].strip()}",
489 }
490 )
492 # Check if the last section is source-related and remove it
493 if structure:
494 last_section = structure[-1]
495 section_name_lower = last_section["name"].lower()
496 source_keywords = [
497 "source",
498 "citation",
499 "reference",
500 "bibliography",
501 ]
503 # Only check the last section for source-related content
504 if any(
505 keyword in section_name_lower for keyword in source_keywords
506 ):
507 logger.info(
508 f"Removed source-related last section: {last_section['name']}"
509 )
510 structure = structure[:-1]
512 return structure
514 def _truncate_at_sentence_boundary(self, text: str, max_chars: int) -> str:
515 """Truncate text at a sentence boundary to preserve readability.
517 Attempts to cut at the last sentence-ending punctuation (.!?) before
518 the limit. If no suitable boundary is found within 80% of the limit,
519 falls back to hard truncation.
521 Args:
522 text: Text to truncate.
523 max_chars: Maximum characters allowed.
525 Returns:
526 Truncated text with ``[...truncated]`` marker if truncation
527 occurred, otherwise the original text unchanged.
528 """
529 if len(text) <= max_chars:
530 return text
532 truncated = text[:max_chars]
534 # Look for sentence boundaries (. ! ?) followed by space or newline
535 # Search backwards from the end for the last complete sentence
536 last_sentence_end = -1
537 for i in range(len(truncated) - 1, -1, -1):
538 if truncated[i] in ".!?" and (
539 i + 1 >= len(truncated) or truncated[i + 1] in " \n"
540 ):
541 last_sentence_end = i + 1
542 break
544 # Only use sentence boundary if it preserves at least 80% of content
545 min_acceptable = int(max_chars * 0.8)
546 if last_sentence_end > min_acceptable:
547 return truncated[:last_sentence_end] + "\n[...truncated]"
549 # Fall back to hard truncation
550 return truncated + "\n[...truncated]"
552 @staticmethod
553 def _normalize_heading_text(text: str) -> str:
554 """Normalize heading text for fuzzy comparison against subsection names.
556 Strips markdown bold markers, leading Roman-numeral / alphabetic /
557 numeric enumeration prefixes (``II.``, ``A.``, ``1.``), collapses
558 whitespace, and lowercases. Used only for matching — never written
559 back into the report.
560 """
561 cleaned = text.strip()
562 # Drop surrounding bold/italic markers the LLM wraps titles in
563 # (``**Title**``, ``*Title*``, ``__Title__``). Avoid regex here —
564 # CodeQL flags ``^[*_]+|[*_]+$`` as polynomial on long ``*`` runs.
565 cleaned = cleaned.strip("*_").strip()
566 cleaned = re.sub(
567 r"^(?:"
568 r"[IVXLCDM]+\.|" # Roman numerals: II. III. IV. (uppercase only to avoid "Mix." false strip)
569 r"[A-Z]\.|" # Single letters: A. B.
570 r"\d+\." # Arabic numerals: 1. 2.
571 r")\s+",
572 "",
573 cleaned,
574 )
575 # Normalise curly/smart quotes and dashes so "Author's" (U+2019)
576 # matches "Author's" (ASCII apostrophe) and en/em dashes collapse
577 # to a plain hyphen for comparison.
578 cleaned = cleaned.translate(
579 str.maketrans(
580 "\u2018\u2019\u201c\u201d\u2013\u2014\u2212", "''\"\"---"
581 )
582 )
583 return re.sub(r"\s+", " ", cleaned).strip().lower()
585 @classmethod
586 def _heading_restates_name(cls, heading_text: str, name: str) -> bool:
587 """Return True if *heading_text* is a restatement of *name*.
589 Handles exact matches, prefix matches (heading extends the name
590 with a colon/dash subtitle), and the reverse (name is a longer
591 form of a short heading). Empty names never match.
592 """
593 if not name or not heading_text:
594 return False
595 norm_heading = cls._normalize_heading_text(heading_text)
596 norm_name = cls._normalize_heading_text(name)
597 if not norm_heading or not norm_name: 597 ↛ 598line 597 didn't jump to line 598 because the condition on line 597 was never true
598 return False
599 if norm_heading == norm_name:
600 return True
601 # Heading extends the subsection name with a subtitle (e.g. "Name: subtitle")
602 # Delimiters only: bare space prefixes (e.g. "Introduction to …")
603 # are intentional non-matches. En/em dashes are already folded to
604 # ASCII "-" by _normalize_heading_text.
605 if norm_heading.startswith(norm_name):
606 rest = norm_heading[len(norm_name) :].lstrip()
607 if rest and rest[0] in (":", "-", "("):
608 return True
609 # Name extends a short heading with a subtitle.
610 if norm_name.startswith(norm_heading): 610 ↛ 611line 610 didn't jump to line 611 because the condition on line 610 was never true
611 rest = norm_name[len(norm_heading) :].lstrip()
612 if rest and rest[0] in (":", "-", "("):
613 return True
614 return False
616 def _strip_leading_heading(
617 self,
618 content: str,
619 subsection_name: str,
620 sibling_subsection_names: Optional[List[str]] = None,
621 ) -> str:
622 """Drop a single leading heading line only when it restates subsection or sibling name."""
623 leading = _LEADING_HEADING_RE.match(content)
624 if not leading:
625 return content
627 fence_spans = _get_code_fence_spans(content)
628 if _is_in_spans(leading.start(), fence_spans): 628 ↛ 629line 628 didn't jump to line 629 because the condition on line 628 was never true
629 return content
631 heading_text = re.sub(r"^\s*#{1,6}[ \t]+", "", leading.group(0)).strip()
632 # Subsection name: allow subtitle extensions (Name: subtitle)
633 if self._heading_restates_name(heading_text, subsection_name):
634 end_pos = leading.end()
635 while end_pos < len(content) and content[end_pos] in "\r\n":
636 end_pos += 1
637 return content[end_pos:]
638 # Siblings: require exact normalized equality (avoid over-strip of
639 # "### <Sibling>: Subtitle" organizers)
640 norm_heading = self._normalize_heading_text(heading_text)
641 for sibling in sibling_subsection_names or []:
642 if sibling and norm_heading == self._normalize_heading_text( 642 ↛ 641line 642 didn't jump to line 641 because the condition on line 642 was always true
643 sibling
644 ):
645 end_pos = leading.end()
646 while end_pos < len(content) and content[end_pos] in "\r\n":
647 end_pos += 1
648 return content[end_pos:]
649 return content
651 def _strip_leading_italic_purpose(
652 self,
653 content: str,
654 purpose: Optional[str] = None,
655 ) -> str:
656 """Drop a single leading italic purpose statement if it mirrors purpose or boilerplate."""
657 leading_italic = _LEADING_ITALIC_PURPOSE_RE.match(content)
658 if not leading_italic:
659 return content
661 fence_spans = _get_code_fence_spans(content)
662 if _is_in_spans(leading_italic.start(), fence_spans): 662 ↛ 663line 662 didn't jump to line 663 because the condition on line 662 was never true
663 return content
665 italic_raw = leading_italic.group(0).strip()
666 italic_text = italic_raw.strip("_*").strip()
667 should_strip = False
669 if purpose:
670 norm_italic = self._normalize_heading_text(italic_text)
671 norm_purpose = self._normalize_heading_text(purpose)
672 # Guard against degenerate purpose that normalizes to "" (e.g. "***")
673 # — startswith("") is always True and would strip any italic line.
674 if norm_italic and norm_purpose: 674 ↛ 683line 674 didn't jump to line 683 because the condition on line 674 was always true
675 if (
676 norm_italic == norm_purpose
677 or norm_italic.startswith(norm_purpose)
678 or norm_purpose.startswith(norm_italic)
679 or self._heading_restates_name(italic_text, purpose)
680 ):
681 should_strip = True
683 if not should_strip:
684 norm_italic = self._normalize_heading_text(italic_text)
685 if norm_italic: 685 ↛ 777line 685 didn't jump to line 777 because the condition on line 685 was always true
686 # Narrow unconditional boilerplate markers — always safe to strip
687 if norm_italic.startswith(
688 (
689 "purpose:",
690 "scope:",
691 "this subsection ",
692 "this section ",
693 )
694 ):
695 should_strip = True
696 # Verb-prefix boilerplate: check for relevance to purpose to avoid
697 # eating legitimate epigraphs like "_To review the evidence is to understand the past._"
698 # when purpose is unrelated (S2).
699 elif purpose is not None and norm_italic.startswith(
700 (
701 "to summarize",
702 "to summarise",
703 "to clarify",
704 "to examine",
705 "to outline",
706 "to provide",
707 "to describe",
708 "to detail",
709 "to analyze",
710 "to analyse",
711 "to explore",
712 "to present",
713 "to discuss",
714 "to evaluate",
715 "to investigate",
716 "to review",
717 "to assess",
718 "aims to",
719 "designed to",
720 )
721 ):
722 norm_purpose = self._normalize_heading_text(purpose)
723 if norm_purpose: 723 ↛ 777line 723 didn't jump to line 777 because the condition on line 723 was always true
724 _stop_words = {
725 "to",
726 "the",
727 "and",
728 "for",
729 "that",
730 "this",
731 "with",
732 "from",
733 "summarize",
734 "summarise",
735 "clarify",
736 "examine",
737 "outline",
738 "provide",
739 "describe",
740 "detail",
741 "analyze",
742 "analyse",
743 "explore",
744 "present",
745 "discuss",
746 "evaluate",
747 "investigate",
748 "review",
749 "assess",
750 "aims",
751 "designed",
752 "section",
753 "subsection",
754 }
755 purpose_words = {
756 w
757 for w in re.findall(
758 r"\b[a-z0-9]{3,}\b", norm_purpose.lower()
759 )
760 if w not in _stop_words
761 }
762 italic_words = {
763 w
764 for w in re.findall(
765 r"\b[a-z0-9]{3,}\b", norm_italic.lower()
766 )
767 if w not in _stop_words
768 }
769 if ( 769 ↛ 775line 769 didn't jump to line 775 because the condition on line 769 was never true
770 self._heading_restates_name(italic_text, purpose)
771 or norm_italic.startswith(norm_purpose)
772 or norm_purpose.startswith(norm_italic)
773 or bool(purpose_words & italic_words)
774 ):
775 should_strip = True
777 if should_strip:
778 end_pos = leading_italic.end()
779 while end_pos < len(content) and content[end_pos] in "\r\n":
780 end_pos += 1
781 return content[end_pos:]
782 return content
784 def _strip_embedded_bibliographies(self, content: str) -> str:
785 """Remove every embedded bibliography block outside code blocks.
787 Only removes a heading that looks like a bibliography *and* whose
788 following block looks like a citation list — this prevents
789 substantive headings such as "### Sources of Bias..." or
790 "### Sources. Data Collection Methodology" from being deleted with
791 their analysis body (R1). Consecutive bibliography headings no longer
792 consume intervening prose (R8).
793 """
794 fence_spans = _get_code_fence_spans(content)
795 pieces: List[str] = []
796 cursor = 0
797 bib_removed = False
799 for match in _BIBLIOGRAPHY_HEADING_RE.finditer(content):
800 if _is_in_spans(match.start(), fence_spans):
801 continue
803 matched_heading_text = match.group(0).lstrip()
804 is_atx_heading = matched_heading_text.startswith("#")
805 bib_line_re = (
806 _CITATION_LINE_RE
807 if is_atx_heading
808 else _PSEUDO_BIB_CITATION_LINE_RE
809 )
811 after_heading = match.end()
812 next_heading_start = None
813 next_heading_is_bib = False
814 for nh in _NEXT_HEADING_RE.finditer(content, after_heading):
815 if not _is_in_spans(nh.start(), fence_spans): 815 ↛ 814line 815 didn't jump to line 814 because the condition on line 815 was always true
816 next_heading_start = nh.start()
817 # Peek if the next heading itself is a bibliography heading
818 # — if so, we will trim the current block to its citation
819 # list end rather than to the heading (R8).
820 heading_line = content[nh.start() :].split("\n", 1)[0]
821 if _BIBLIOGRAPHY_HEADING_RE.match(heading_line): 821 ↛ 822line 821 didn't jump to line 822 because the condition on line 821 was never true
822 next_heading_is_bib = True
823 break
825 block_end = (
826 next_heading_start
827 if next_heading_start is not None
828 else len(content)
829 )
830 block_text = content[after_heading:block_end]
832 # If the block immediately begins with a code fence, the heading is
833 # introducing fenced example/code content, not a bibliography.
834 offset = 0
835 first_non_empty_start = None
836 for ln in block_text.splitlines(keepends=True): 836 ↛ 842line 836 didn't jump to line 842 because the loop on line 836 didn't complete
837 if ln.strip():
838 first_non_empty_start = after_heading + offset
839 break
840 offset += len(ln)
842 if first_non_empty_start is not None and _is_in_spans(
843 first_non_empty_start, fence_spans
844 ):
845 continue
847 # Body-shape safeguard: only treat as bibliography if the block
848 # contains citation-like lines. Require at least one citation
849 # marker, or >30% of non-empty lines look like citations, to
850 # avoid deleting substantive analysis headings that happen to
851 # match the label grammar (e.g. "### Sources for the Analysis"
852 # with prose body).
853 non_empty_lines = []
854 line_offset = 0
855 for ln in block_text.splitlines(keepends=True):
856 line_start = after_heading + line_offset
857 if ln.strip() and not _is_in_spans(line_start, fence_spans):
858 non_empty_lines.append(ln)
859 line_offset += len(ln)
860 if not non_empty_lines: 860 ↛ 862line 860 didn't jump to line 862 because the condition on line 860 was never true
861 # Empty block — treat as bibliography (heading with no body)
862 is_bib_block = True
863 else:
864 # Intervening prose safeguard: a bibliography heading must introduce
865 # citations directly (or via an immediate note / 1-line lead-in).
866 # If substantive prose precedes citations, the heading is introducing
867 # substantive analysis, not an embedded bibliography.
868 first_line = non_empty_lines[0]
869 first_is_bib = (
870 bib_line_re.match(first_line)
871 or _BIB_NOTE_RE.match(first_line)
872 or _HR_LINE_RE.match(first_line)
873 )
874 if not first_is_bib:
875 # Check if it's a short 1-line lead-in immediately followed by citations
876 second_is_citation = len(
877 non_empty_lines
878 ) > 1 and bib_line_re.match(non_empty_lines[1])
879 is_lead_in = (
880 len(first_line.strip()) < 120
881 and second_is_citation
882 and bool(
883 re.search(
884 r"\b(?:sources?|references?|consulted|cited|following|basis)\b",
885 first_line,
886 re.IGNORECASE,
887 )
888 )
889 )
890 has_early_note = any(
891 _BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
892 )
893 if not (is_lead_in or has_early_note):
894 continue
896 citation_lines = sum(
897 1 for ln in non_empty_lines if bib_line_re.match(ln)
898 )
899 # Require explicit italic note boilerplate (e.g. "*(Note: ... bibliography ...)*")
900 # rather than matching arbitrary prose containing "bibliography" (S1).
901 has_bib_note = any(
902 _BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
903 )
904 if citation_lines == 0 and not has_bib_note: 904 ↛ 906line 904 didn't jump to line 906 because the condition on line 904 was never true
905 # No citation shape and no italic bib note — preserve heading + body
906 continue
907 # If citation lines present, require they are not a tiny minority
908 # unless block is short (e.g. "- item" single line)
909 if citation_lines == 0:
910 is_bib_block = has_bib_note
911 elif len(non_empty_lines) <= 3:
912 is_bib_block = citation_lines >= 1
913 else:
914 is_bib_block = (
915 citation_lines / len(non_empty_lines)
916 ) >= 0.3 or citation_lines >= 2
918 if not is_bib_block: 918 ↛ 919line 918 didn't jump to line 919 because the condition on line 918 was never true
919 continue
921 # Passed safeguard — this is a real bibliography block to remove
922 bib_removed = True
923 block_start = match.start()
924 prefix = content[cursor:block_start]
925 prefix_lines = prefix.splitlines(keepends=True)
926 cut_idx = len(prefix)
927 curr_offset = len(prefix)
928 found_delimiter = False
929 for pline in reversed(prefix_lines):
930 line_start = curr_offset - len(pline)
931 abs_pos = cursor + line_start
932 if _is_in_spans(abs_pos, fence_spans):
933 break
934 pstripped = pline.strip()
935 if not pstripped:
936 curr_offset = line_start
937 continue
938 if _EMPTY_HEADING_OR_HR_RE.match(pstripped):
939 found_delimiter = True
940 cut_idx = line_start
941 curr_offset = line_start
942 else:
943 break
944 if found_delimiter and cut_idx < len(prefix):
945 block_start = cursor + cut_idx
947 pieces.append(content[cursor:block_start])
949 # R8: consecutive bib headings — end at citation list, not at next bib heading
950 if next_heading_is_bib: 950 ↛ 952line 950 didn't jump to line 952 because the condition on line 950 was never true
951 # Find end of citation list within block_text
952 lines = block_text.splitlines(keepends=True)
953 offset = 0
954 last_citation_end = 0
955 for ln in lines:
956 line_start = after_heading + offset
957 if not _is_in_spans(line_start, fence_spans) and (
958 bib_line_re.match(ln)
959 or (_BIB_NOTE_RE.match(ln) and offset < 300)
960 ):
961 last_citation_end = offset + len(ln)
962 # Also keep consecutive citation lines; stop at first
963 # non-citation prose that is not a blank/bib note?
964 offset += len(ln)
965 # Also include trailing blank lines after citations, but not
966 # intervening prose before next bib heading
967 if last_citation_end > 0:
968 # Advance cursor to after the citation list (preserve
969 # intervening prose between two bib headings)
970 cursor = after_heading + last_citation_end
971 # Skip following blank lines / HRs but not prose
972 while cursor < block_end and content[cursor] in " \t\r\n-":
973 # Only skip whitespace/HRs, stop at prose
974 # Peek next non-whitespace chunk
975 nxt = content[cursor:].lstrip(" \t\r\n")
976 if nxt.startswith("-") and nxt[1:2] in " \t":
977 # Another HR-like line, skip it
978 cursor += len(content[cursor:]) - len(nxt)
979 # consume the HR line
980 eol = content.find("\n", cursor)
981 cursor = eol + 1 if eol != -1 else len(content)
982 else:
983 break
984 if cursor >= block_end:
985 break
986 # Ensure we don't overshoot into next bib heading's prefix;
987 # if cursor is before next_heading_start, keep it, else
988 # fall back to next_heading_start
989 if cursor > block_end:
990 cursor = block_end
991 else:
992 cursor = block_end
993 else:
994 # End the removal at the citation list rather than at the end
995 # of the block, so a table, list or paragraph the model wrote
996 # after its bibliography survives the strip.
997 offset = 0
998 citation_end = 0
999 non_empty_seen = 0
1000 trailing_content = False
1001 for ln in block_text.splitlines(keepends=True):
1002 line_start = after_heading + offset
1003 substantive = bool(ln.strip())
1004 outside_fence = not _is_in_spans(line_start, fence_spans)
1005 non_empty_seen += substantive and outside_fence
1006 # The note window is the classifier's first-three-non-empty-
1007 # lines rule, so a block accepted as a bibliography always
1008 # has a qualifying line here and cannot fall through.
1009 is_bib_line = bool(
1010 outside_fence
1011 and (
1012 bib_line_re.match(ln)
1013 or (_BIB_NOTE_RE.match(ln) and non_empty_seen <= 3)
1014 or _HR_LINE_RE.match(ln)
1015 )
1016 )
1017 if citation_end and substantive and not is_bib_line:
1018 trailing_content = True
1019 break
1020 offset += len(ln)
1021 if substantive and is_bib_line:
1022 citation_end = offset
1023 if not trailing_content:
1024 # Nothing substantive follows, so the whole block is the
1025 # bibliography and is removed as before.
1026 citation_end = len(block_text)
1028 cursor = after_heading + citation_end
1030 if bib_removed:
1031 pieces.append(content[cursor:])
1032 content = "".join(pieces)
1033 return self._strip_trailing_citation_blocks(content)
1035 def _strip_trailing_citation_blocks(self, content: str) -> str:
1036 """Remove unheaded trailing citation blocks or concatenated citations.
1038 When the LLM is instructed not to include a sources section heading,
1039 some models omit the heading entirely but append a list or block of
1040 citations at the end of the response (e.g. consecutive lines of
1041 `[[111]](/library/...)` or a concatenated line of citations).
1043 This defensive helper scans backwards from the end of the content
1044 outside code fences and strips trailing citation blocks while
1045 preserving all substantive prose, tables, code fences, reference-style
1046 link definitions, further-reading URLs, quiz answer keys, and CJK notes.
1047 """
1048 if not content:
1049 return content
1051 fence_spans = _get_code_fence_spans(content)
1052 lines = content.splitlines(keepends=True)
1053 if not lines: 1053 ↛ 1054line 1053 didn't jump to line 1054 because the condition on line 1053 was never true
1054 return content
1056 line_offsets: List[int] = []
1057 curr = 0
1058 for ln in lines:
1059 line_offsets.append(curr)
1060 curr += len(ln)
1062 # Skip trailing empty / whitespace lines
1063 cut_idx = len(lines)
1064 while cut_idx > 0 and not lines[cut_idx - 1].strip():
1065 cut_idx -= 1
1067 if cut_idx == 0: 1067 ↛ 1068line 1067 didn't jump to line 1068 because the condition on line 1067 was never true
1068 return content
1070 # Never strip inside code fences
1071 if _is_in_spans(line_offsets[cut_idx - 1], fence_spans):
1072 return content
1074 citation_lines_count = 0
1075 concatenated_lines_count = 0
1076 bib_heading_preceded = False
1077 idx = cut_idx
1079 while idx > 0: 1079 ↛ 1148line 1079 didn't jump to line 1148 because the condition on line 1079 was always true
1080 offset = line_offsets[idx - 1]
1081 if _is_in_spans(offset, fence_spans): 1081 ↛ 1082line 1081 didn't jump to line 1082 because the condition on line 1081 was never true
1082 break
1084 line_str = lines[idx - 1].strip()
1085 if not line_str:
1086 idx -= 1
1087 continue
1089 # Reference-style link definitions ([1]: https://...) must NEVER be stripped
1090 if _MD_LINK_DEF_RE.match(line_str):
1091 break
1093 # Plain numbered or bulleted list items must NEVER be stripped as unheaded citations
1094 if _ORDERED_OR_BULLET_LIST_RE.match(line_str):
1095 break
1097 # Concatenated citation clusters (e.g. [[217]][[218]] or [[1]](/a), [2], [[3]])
1098 if _CONCATENATED_CITATIONS_LINE_RE.match(line_str):
1099 # Must contain double brackets [[...]], cite tags, or adjacent bracket tokens;
1100 # exclude plain numbered markdown links like [1](url) [2](url)
1101 if (
1102 "[[" in line_str
1103 or "[cite" in line_str.lower()
1104 or bool(re.search(r"\]\s*\[|】\s*[【\[]|\]\s*【", line_str))
1105 ):
1106 concatenated_lines_count += 1
1107 idx -= 1
1108 continue
1109 break
1111 if _UNHEADED_CITATION_ITEM_RE.match(line_str):
1112 citation_lines_count += 1
1113 idx -= 1
1114 continue
1116 # Support indented continuation lines within a citation entry.
1117 # When scanning backwards, find the preceding non-indented line.
1118 if lines[idx - 1].startswith((" ", "\t")):
1119 p = idx - 2
1120 while p >= 0 and lines[p].startswith((" ", "\t")): 1120 ↛ 1121line 1120 didn't jump to line 1121 because the condition on line 1120 was never true
1121 p -= 1
1122 if p >= 0 and _UNHEADED_CITATION_ITEM_RE.match( 1122 ↛ 1128line 1122 didn't jump to line 1128 because the condition on line 1122 was always true
1123 lines[p].strip()
1124 ):
1125 idx -= 1
1126 continue
1128 if _BIB_NOTE_RE.match(line_str): 1128 ↛ 1129line 1128 didn't jump to line 1129 because the condition on line 1128 was never true
1129 idx -= 1
1130 continue
1132 # Check if preceded by an explicit bibliography heading or label
1133 if citation_lines_count > 0 or concatenated_lines_count > 0:
1134 if _BIBLIOGRAPHY_HEADING_RE.match(line_str): 1134 ↛ 1135line 1134 didn't jump to line 1135 because the condition on line 1134 was never true
1135 bib_heading_preceded = True
1136 idx -= 1
1137 continue
1138 if _EMPTY_HEADING_OR_HR_RE.match(line_str):
1139 idx -= 1
1140 continue
1142 break
1144 # Only strip if:
1145 # 1. Found a concatenated citation cluster
1146 # 2. Preceded by an explicit bibliography heading/label and found >= 1 citation items
1147 # 3. Found >= 2 citation items in an unheaded block (bare --- does NOT arm single items)
1148 should_strip = False
1149 if concatenated_lines_count > 0:
1150 should_strip = True
1151 elif bib_heading_preceded and citation_lines_count >= 1: 1151 ↛ 1152line 1151 didn't jump to line 1152 because the condition on line 1151 was never true
1152 should_strip = True
1153 elif citation_lines_count >= 2:
1154 should_strip = True
1156 if should_strip:
1157 # Also consume any preceding empty headings, labels, or HRs
1158 while idx > 0: 1158 ↛ 1172line 1158 didn't jump to line 1172 because the condition on line 1158 was always true
1159 offset = line_offsets[idx - 1]
1160 if _is_in_spans(offset, fence_spans): 1160 ↛ 1161line 1160 didn't jump to line 1161 because the condition on line 1160 was never true
1161 break
1162 pstripped = lines[idx - 1].strip()
1163 if not pstripped: 1163 ↛ 1164line 1163 didn't jump to line 1164 because the condition on line 1163 was never true
1164 idx -= 1
1165 continue
1166 if _EMPTY_HEADING_OR_HR_RE.match( 1166 ↛ 1169line 1166 didn't jump to line 1169 because the condition on line 1166 was never true
1167 pstripped
1168 ) or _BIBLIOGRAPHY_HEADING_RE.match(pstripped):
1169 idx -= 1
1170 else:
1171 break
1172 return "".join(lines[:idx])
1174 return content
1176 def _strip_subsection_boilerplate(
1177 self,
1178 content: str,
1179 subsection_name: str,
1180 section_name: str,
1181 sibling_subsection_names: Optional[List[str]] = None,
1182 purpose: Optional[str] = None,
1183 ) -> str:
1184 """Strip boilerplate the LLM tends to emit around subsection content.
1186 Small and/or quantized local models routinely produce three artefacts
1187 that pollute the rendered report even when the OUTPUT FORMAT RULES
1188 spelled out in the subsection prompt explicitly forbid them:
1190 1. A redundant leading heading (stripped by :meth:`_strip_leading_heading`)
1191 2. An italic purpose statement (stripped by :meth:`_strip_leading_italic_purpose`)
1192 3. An embedded bibliography block (stripped by :meth:`_strip_embedded_bibliographies`)
1194 This helper normalises the per-subsection content so the rendered
1195 report stays clean even when the model ignored the prompt. It is a
1196 defensive cleanup, not a substitute for the prompt rules.
1198 Args:
1199 content: Raw ``current_knowledge`` returned for one subsection.
1200 subsection_name: The subsection's name (used for matching and
1201 logging).
1202 section_name: The parent section's name (used for logging).
1203 sibling_subsection_names: Optional names of other subsections
1204 in the same section. When provided, a leading heading that
1205 restates a *sibling's* name (context-bleed) is also stripped.
1206 purpose: Optional purpose string for the subsection. Used to
1207 verify if a leading italic line mirrors the subsection purpose.
1209 Returns:
1210 ``content`` with a redundant leading heading removed when it
1211 restates this or a sibling subsection name, a leading
1212 italic-purpose statement removed, and any embedded bibliography
1213 blocks removed. Returns content unchanged if no artefacts are
1214 found, aside from collapsing 3+ blank lines to 2 and
1215 normalising trailing whitespace (collapsed outside code fences
1216 only, so fenced blocks are never rewritten).
1217 """
1218 if not content:
1219 return content
1221 original_len = len(content)
1222 new_content = self._strip_leading_heading(
1223 content, subsection_name, sibling_subsection_names
1224 )
1225 new_content = self._strip_leading_italic_purpose(new_content, purpose)
1226 pre_bib_len = len(new_content)
1227 new_content = self._strip_embedded_bibliographies(new_content)
1228 pre_trailing_len = len(new_content)
1229 new_content = self._strip_trailing_citation_blocks(new_content)
1230 bib_chars_removed = (pre_bib_len - pre_trailing_len) + (
1231 pre_trailing_len - len(new_content)
1232 )
1234 # R2: fence-aware whitespace collapse — never rewrite inside code fences
1235 fence_spans = _get_code_fence_spans(new_content)
1236 if not fence_spans:
1237 collapsed = re.sub(r"\n{3,}", "\n\n", new_content).strip()
1238 else:
1239 parts: List[str] = []
1240 last = 0
1241 for s, e in sorted(fence_spans):
1242 # collapse outside fence
1243 parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:s]))
1244 # keep fence content verbatim
1245 parts.append(new_content[s:e])
1246 last = e
1247 parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:]))
1248 collapsed = "".join(parts).strip()
1249 new_content = collapsed
1250 if new_content:
1251 new_content += "\n"
1253 if len(new_content) != original_len:
1254 # R5: make silent deletions observable
1255 if not new_content.strip() and content.strip():
1256 logger.warning(
1257 "Stripped subsection boilerplate emptied non-trivial content for "
1258 f"'{section_name} > {subsection_name}': "
1259 f"{original_len} -> {len(new_content)} chars — "
1260 "check for over-strip (R1/R3 regression)"
1261 )
1262 elif len(new_content) < 0.5 * original_len:
1263 # If the truncation below 50% was driven by stripping an
1264 # embedded bibliography (where pre-bibliography content was
1265 # >= 50% and bib removal did the rest), this is expected
1266 # cleanup rather than an unexpected over-strip.
1267 if (
1268 bib_chars_removed > 0
1269 and pre_bib_len >= 0.5 * original_len
1270 and (original_len - pre_bib_len) < 0.25 * original_len
1271 ):
1272 logger.info(
1273 "Stripped embedded bibliography from subsection "
1274 f"'{section_name} > {subsection_name}': "
1275 f"{original_len} -> {len(new_content)} chars "
1276 "(expected cleanup; warning demoted to INFO)"
1277 )
1278 else:
1279 logger.warning(
1280 "Stripped subsection boilerplate heavily truncated content for "
1281 f"'{section_name} > {subsection_name}': "
1282 f"{original_len} -> {len(new_content)} chars"
1283 )
1284 else:
1285 logger.debug(
1286 "Stripped subsection boilerplate for "
1287 f"'{section_name} > {subsection_name}': "
1288 f"{original_len} -> {len(new_content)} chars"
1289 )
1290 return new_content
1292 def _build_previous_context(self, accumulated_findings: List[str]) -> str:
1293 """Build context block from previously generated sections.
1295 Creates a formatted context block containing content from the last
1296 N sections (defined by self.max_context_sections) with explicit instructions
1297 not to repeat this content. Context is truncated if it exceeds
1298 self.max_context_chars to stay safe for smaller local models.
1300 Args:
1301 accumulated_findings: List of previously generated section content,
1302 each formatted as "[Section > Subsection]\\n{content}"
1304 Returns:
1305 Formatted context block with delimiters, or empty string if no
1306 previous findings exist
1307 """
1308 if not accumulated_findings:
1309 return ""
1311 recent_findings = accumulated_findings[-self.max_context_sections :]
1312 previous_context = "\n\n---\n\n".join(recent_findings)
1314 # Truncate at sentence boundary if too long
1315 if len(previous_context) > self.max_context_chars:
1316 previous_context = self._truncate_at_sentence_boundary(
1317 previous_context, self.max_context_chars
1318 )
1320 return (
1321 f"\n\n=== CONTENT ALREADY WRITTEN (DO NOT REPEAT) ===\n"
1322 f"{previous_context}\n"
1323 f"=== END OF PREVIOUS CONTENT ===\n\n"
1324 f"CRITICAL: The above content has already been written. Do NOT repeat "
1325 f"these points, examples, or explanations. Focus on NEW information "
1326 f"not covered above.\n"
1327 )
1329 def _research_and_generate_sections(
1330 self,
1331 initial_findings: Dict,
1332 structure: List[Dict],
1333 query: str,
1334 progress_callback=None,
1335 ) -> Dict[str, str]:
1336 """Research and generate content for each section in one step.
1338 This method processes sections sequentially, accumulating generated
1339 content as it goes. For each new section/subsection, it passes context
1340 from the last few previously generated sections to help the LLM avoid
1341 repetition.
1343 The context accumulation mechanism:
1344 - Tracks all generated content in accumulated_findings list
1345 - Before generating each section, builds context from recent findings
1346 - Uses self.max_context_sections (configurable, default: 3) to limit context size
1347 - Truncates context to self.max_context_chars (configurable, default: 4000) for safety
1348 - Includes explicit "DO NOT REPEAT" instructions with actual content
1350 Args:
1351 initial_findings: Results from initial research phase, may contain
1352 questions_by_iteration to preserve search continuity
1353 structure: List of section definitions, each with name and subsections
1354 query: Original user query for context
1356 Returns:
1357 Dict mapping section names to their generated markdown content
1358 """
1359 sections = {}
1361 # Accumulate content from previous sections to avoid repetition
1362 accumulated_findings: List[str] = []
1364 # Count total subsections for progress tracking
1365 total_subsections = sum(
1366 max(len(section.get("subsections", [])), 1) for section in structure
1367 )
1368 completed_subsections = 0
1370 # Preserve questions from initial research to avoid repetition
1371 # This follows the same pattern as citation tracking (all_links_of_system)
1372 existing_questions = initial_findings.get("questions_by_iteration", {})
1373 if existing_questions:
1374 # Set questions on both search system and its strategy
1375 if hasattr(self.search_system, "questions_by_iteration"): 1375 ↛ 1381line 1375 didn't jump to line 1381 because the condition on line 1375 was always true
1376 self.search_system.questions_by_iteration = (
1377 existing_questions.copy()
1378 )
1380 # More importantly, set it on the strategy which actually uses it
1381 if hasattr(self.search_system, "strategy") and hasattr( 1381 ↛ 1391line 1381 didn't jump to line 1391 because the condition on line 1381 was always true
1382 self.search_system.strategy, "questions_by_iteration"
1383 ):
1384 self.search_system.strategy.questions_by_iteration = (
1385 existing_questions.copy()
1386 )
1387 logger.info(
1388 f"Initialized strategy with {len(existing_questions)} iterations of previous questions"
1389 )
1391 for i, section in enumerate(structure, 1):
1392 logger.info(f"Processing section: {section['name']}")
1393 section_content = []
1395 section_content.append(f"# {i}. {section['name']}\n")
1397 # If section has no subsections, create one from the section itself
1398 if not section["subsections"]:
1399 # Parse section name for purpose
1400 if "|" in section["name"]:
1401 parts = section["name"].split("|", 1)
1402 section["subsections"] = [
1403 {"name": parts[0].strip(), "purpose": parts[1].strip()}
1404 ]
1405 else:
1406 # No purpose provided - use section name as subsection
1407 section["subsections"] = [
1408 {
1409 "name": section["name"],
1410 "purpose": f"Provide comprehensive content for {section['name']}",
1411 }
1412 ]
1414 # Process each subsection by directly researching it
1415 for j, subsection in enumerate(section["subsections"], 1):
1416 # Only add subsection header if there are multiple subsections
1417 if len(section["subsections"]) > 1:
1418 section_content.append(f"## {i}.{j} {subsection['name']}\n")
1419 section_content.append(f"_{subsection['purpose']}_\n\n")
1421 # Get other subsections in this section for context
1422 other_subsections = [
1423 f"- {s['name']}: {s['purpose']}"
1424 for s in section["subsections"]
1425 if s["name"] != subsection["name"]
1426 ]
1427 other_subsections_text = (
1428 "\n".join(other_subsections)
1429 if other_subsections
1430 else "None"
1431 )
1433 # Get all other sections for broader context
1434 other_sections = [
1435 f"- {s['name']}"
1436 for s in structure
1437 if s["name"] != section["name"]
1438 ]
1439 other_sections_text = (
1440 "\n".join(other_sections) if other_sections else "None"
1441 )
1443 # Check if this is actually a section-level content (only one subsection, likely auto-created)
1444 is_section_level = len(section["subsections"]) == 1
1446 # Build context from previously generated sections to avoid repetition
1447 previous_context_section = self._build_previous_context(
1448 accumulated_findings
1449 )
1451 # Generate appropriate search query
1452 if is_section_level:
1453 # Section-level prompt - more comprehensive
1454 subsection_query = (
1455 f"Research task: Create comprehensive content for the '{subsection['name']}' section in a report about '{query}'. "
1456 f"Section purpose: {subsection['purpose']} "
1457 f"\n"
1458 f"Other sections in the report:\n{other_sections_text}\n"
1459 f"{previous_context_section}"
1460 f"This is a standalone section requiring comprehensive coverage of its topic. "
1461 f"Provide a thorough exploration that may include synthesis of information from previous sections where relevant. "
1462 f"Include unique insights, specific examples, and concrete data. "
1463 f"Use tables to organize information where applicable. "
1464 f"For conclusion sections: synthesize key findings and provide forward-looking insights. "
1465 f"Build upon the research findings from earlier sections to create a cohesive narrative."
1466 f"{_SUBSECTION_OUTPUT_GUIDANCE}"
1467 )
1468 else:
1469 # Subsection-level prompt - more focused
1470 subsection_query = (
1471 f"Research task: Create content for subsection '{subsection['name']}' in a report about '{query}'. "
1472 f"This subsection's purpose: {subsection['purpose']} "
1473 f"Part of section: '{section['name']}' "
1474 f"\n"
1475 f"Other sections in the report:\n{other_sections_text}\n"
1476 f"\n"
1477 f"Other subsections in this section will cover:\n{other_subsections_text}\n"
1478 f"{previous_context_section}"
1479 f"Focus ONLY on information specific to your subsection's purpose. "
1480 f"Include unique details, specific examples, and concrete data. "
1481 f"Use tables to organize information where applicable. "
1482 f"IMPORTANT: Avoid repeating information that would logically be covered in other sections - focus on what makes this subsection unique. "
1483 f"Previous research exists - find specific angles for this subsection."
1484 f"{_SUBSECTION_OUTPUT_GUIDANCE}"
1485 )
1487 logger.info(
1488 f"Researching subsection: {subsection['name']} with query: {subsection_query}"
1489 )
1491 # Report progress and check for cancellation
1492 if progress_callback:
1493 pct = int(
1494 10
1495 + (completed_subsections / max(total_subsections, 1))
1496 * 80
1497 )
1498 progress_callback(
1499 f"Researching: {section['name']} > {subsection['name']}",
1500 pct,
1501 {
1502 "phase": "report_section_research",
1503 "subsection": subsection["name"],
1504 },
1505 )
1507 # Fix iteration override: modify strategy's settings_snapshot
1508 # which is read dynamically via get_setting()
1509 strategy = self.search_system.strategy
1510 original_iterations = strategy.settings_snapshot.get(
1511 "search.iterations"
1512 )
1513 had_iterations_key = (
1514 "search.iterations" in strategy.settings_snapshot
1515 )
1516 strategy.settings_snapshot["search.iterations"] = 1
1517 # Belt-and-suspenders: also override max_iterations for
1518 # strategies that cache it at __init__ time
1519 original_max_iter = getattr(strategy, "max_iterations", None)
1520 strategy.max_iterations = 1
1521 # If the strategy wraps a delegate (e.g.
1522 # EnhancedContextualFollowUpStrategy), the actual research
1523 # happens inside ``delegate_strategy.analyze_topic`` and the
1524 # delegate has its own cached ``max_iterations`` from
1525 # construction — without this extra override the per-subsection
1526 # cap never reaches the worker and each subsection runs the
1527 # full configured loop instead of one iteration.
1528 delegate = getattr(strategy, "delegate_strategy", None)
1529 original_delegate_max_iter = (
1530 getattr(delegate, "max_iterations", None)
1531 if delegate is not None
1532 else None
1533 )
1534 if delegate is not None and hasattr(delegate, "max_iterations"):
1535 delegate.max_iterations = 1
1537 try:
1538 # Perform search for this subsection
1539 subsection_results = self.search_system.analyze_topic(
1540 subsection_query
1541 )
1542 finally:
1543 # Restore original iteration settings
1544 if had_iterations_key:
1545 strategy.settings_snapshot["search.iterations"] = (
1546 original_iterations
1547 )
1548 else:
1549 strategy.settings_snapshot.pop(
1550 "search.iterations", None
1551 )
1552 if original_max_iter is not None:
1553 strategy.max_iterations = original_max_iter
1554 if delegate is not None and hasattr(
1555 delegate, "max_iterations"
1556 ):
1557 if original_delegate_max_iter is not None: 1557 ↛ 1560line 1557 didn't jump to line 1560 because the condition on line 1557 was always true
1558 delegate.max_iterations = original_delegate_max_iter
1559 else:
1560 delegate.max_iterations = None
1562 completed_subsections += 1
1564 # Add the researched content for this subsection
1565 if subsection_results.get("current_knowledge"):
1566 generated_content = subsection_results["current_knowledge"]
1567 # Strip the boilerplate the LLM tends to emit around
1568 # subsections: a redundant leading heading that mirrors
1569 # the framework's own `## i.j Name` heading, an italic
1570 # purpose statement that mirrors the framework's
1571 # `_<purpose>_` subtitle, and an embedded
1572 # '## Sources' / '## References' bibliography block
1573 # (the framework appends one master '## Sources' to
1574 # the whole report). Small/quantized local models
1575 # routinely ignore the OUTPUT FORMAT RULES in the
1576 # prompt above; this normalisation step guarantees a
1577 # clean rendered output regardless.
1578 sibling_names = [
1579 s["name"]
1580 for s in section["subsections"]
1581 if s["name"] != subsection["name"]
1582 ]
1583 try:
1584 generated_content = self._strip_subsection_boilerplate(
1585 generated_content,
1586 subsection_name=subsection["name"],
1587 section_name=section["name"],
1588 sibling_subsection_names=sibling_names,
1589 purpose=subsection.get("purpose"),
1590 )
1591 except Exception:
1592 logger.exception(
1593 "Boilerplate strip failed for "
1594 f"'{section['name']} > {subsection['name']}' — "
1595 "using raw LLM content"
1596 )
1597 # Fall back to raw content so a cosmetic step never
1598 # aborts the entire multi-section report.
1599 generated_content = (
1600 subsection_results.get("current_knowledge", "")
1601 or ""
1602 )
1603 if generated_content.strip(): 1603 ↛ 1610line 1603 didn't jump to line 1610 because the condition on line 1603 was always true
1604 section_content.append(generated_content)
1605 # Accumulate for context in subsequent sections
1606 accumulated_findings.append(
1607 f"[{section['name']} > {subsection['name']}]\n{generated_content}"
1608 )
1609 else:
1610 section_content.append(
1611 "*Limited information was found for this subsection.*\n"
1612 )
1613 else:
1614 section_content.append(
1615 "*Limited information was found for this subsection.*\n"
1616 )
1618 section_content.append("\n\n")
1620 # Combine all content for this section
1621 sections[section["name"]] = "\n".join(section_content)
1623 return sections
1625 def _generate_sections(
1626 self,
1627 initial_findings: Dict,
1628 _section_research: Dict[str, List[Dict]],
1629 structure: List[Dict],
1630 query: str,
1631 ) -> Dict[str, str]:
1632 """
1633 This method is kept for compatibility but no longer used.
1634 The functionality has been moved to _research_and_generate_sections.
1635 """
1636 return {}
1638 def _format_final_report(
1639 self,
1640 sections: Dict[str, str],
1641 structure: List[Dict],
1642 query: str,
1643 ) -> Dict:
1644 """Format the final report with table of contents and sections."""
1645 # Generate TOC
1646 toc = ["# Table of Contents\n"]
1647 for i, section in enumerate(structure, 1):
1648 toc.append(f"{i}. **{section['name']}**")
1649 if len(section["subsections"]) > 1:
1650 for j, subsection in enumerate(section["subsections"], 1):
1651 toc.append(
1652 f" {i}.{j} {subsection['name']} | _{subsection['purpose']}_"
1653 )
1655 # Combine TOC and sections
1656 report_parts = ["\n".join(toc), ""]
1658 # Add a summary of the research
1659 report_parts.append("# Research Summary")
1660 report_parts.append(
1661 "This report was researched using an advanced search system."
1662 )
1663 report_parts.append(
1664 "Research included targeted searches for each section and subsection."
1665 )
1666 report_parts.append("\n---\n")
1668 # Add each section's content
1669 for section in structure:
1670 if section["name"] in sections:
1671 report_parts.append(sections[section["name"]])
1672 report_parts.append("")
1674 # Format links from search system
1675 # Get utilities module dynamically to avoid circular imports
1676 utilities = importlib.import_module("local_deep_research.utilities")
1677 # Imported here for the same reason, and directly rather than off
1678 # the module object above so the count is computed by the real
1679 # grouping helper.
1680 from .utilities.search_utilities import (
1681 count_distinct_sources,
1682 filter_cited_links,
1683 resolve_uncited_sources_mode,
1684 )
1686 # Body prose BEFORE the Sources tail: citation indices/URLs are
1687 # parsed from this, so uncited accumulated hits stay out of the
1688 # bibliography. Honors report.uncited_sources_mode.
1689 body_prose = "\n\n".join(report_parts)
1690 mode = resolve_uncited_sources_mode()
1691 visible_links = filter_cited_links(
1692 self.search_system.all_links_of_system,
1693 body_prose,
1694 mode=mode,
1695 )
1696 formatted_all_links = (
1697 utilities.search_utilities.format_links_to_markdown(
1698 all_links=visible_links,
1699 )
1700 )
1702 # Create final report with all parts. The Sources tail is
1703 # kept here so in-memory consumers (MCP `generate_report`,
1704 # programmatic API) get the full assembled blob unchanged.
1705 # The DB save site (research_service.py) strips this Sources
1706 # section via format_document_split before persisting, so the
1707 # answer-only invariant on report_content still holds.
1708 final_report_content = body_prose
1709 # Explicit "\n\n" separator: downstream regex consumers
1710 # (_SOURCES_SECTION_PATTERNS in text_optimization/citation_formatter.py
1711 # and _LEGACY_SOURCES_RE in web/services/report_assembly_service.py)
1712 # use line-anchored `re.MULTILINE` matching. Today the trailing
1713 # newlines produced by `"\n\n".join` happen to keep `## Sources`
1714 # at the start of a line, but that is incidental; an explicit
1715 # separator preserves the invariant against future section
1716 # template changes.
1717 #
1718 # The HTML-comment sentinel around the appended `## Sources`
1719 # block lets ``format_document_split`` locate this section
1720 # unambiguously, even when the LLM has emitted its own
1721 # `## Sources` header earlier in the prose. The legacy regex
1722 # patterns would otherwise match the first `## Sources` they
1723 # find and could over-strip a multi-section report where the
1724 # LLM happened to include an inline sources block. Without the
1725 # sentinel, a 1380-source run legitimately pushes the
1726 # answer/sources ratio below the 50% safety threshold and the
1727 # splitter logs a misleading "over-stripped" warning.
1728 # The sentinel is imported at module top; no circular-import
1729 # concern at use time.
1730 final_report_content += (
1731 f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
1732 f"## Sources\n\n{formatted_all_links}"
1733 )
1735 # Create metadata dictionary
1736 metadata = {
1737 "generated_at": datetime.now(UTC).isoformat(),
1738 # Count the same visible, canonically grouped source set that
1739 # the Sources block above renders. Document-level: one document
1740 # cited via several chunk anchors still counts once even though
1741 # it renders several bibliography lines (distinct sources <=
1742 # bibliography lines <= citation indices) — so a count below
1743 # the rendered line total is expected, not a bug.
1744 "initial_sources": count_distinct_sources(visible_links),
1745 "sections_researched": len(structure),
1746 "searches_per_section": self.searches_per_section,
1747 "query": query,
1748 }
1750 # Return both content and metadata
1751 return {"content": final_report_content, "metadata": metadata}
1753 def _generate_error_report(self, query: str, error_msg: str) -> str:
1754 return f"=== ERROR REPORT ===\nQuery: {query}\nError: {error_msg}"