Coverage for src/local_deep_research/utilities/search_utilities.py: 91%
746 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1import re
2from typing import Dict, List, Optional, Set, Tuple
3from urllib.parse import urlsplit
5from loguru import logger
7from local_deep_research.text_optimization.citation_formatter import (
8 is_line_breaking_char,
9 LDR_APPENDED_SOURCES_SENTINEL,
10)
11from .url_utils import (
12 CHUNK_DISPLAY_KEY,
13 FETCHED_TITLE_KEY,
14 _is_wayback_replay_url,
15 canonical_url_key,
16 library_display_url,
17 preferred_chunk_display,
18)
21LANGUAGE_CODE_MAP = {
22 "english": "en",
23 "french": "fr",
24 "german": "de",
25 "spanish": "es",
26 "italian": "it",
27 "japanese": "ja",
28 "chinese": "zh",
29 "hindi": "hi",
30 "arabic": "ar",
31 "bengali": "bn",
32 "portuguese": "pt",
33 "russian": "ru",
34 "korean": "ko",
35}
38def remove_think_tags(text: str) -> str:
39 # NOTE: Fresh LLM responses from get_llm() are already <think>-stripped
40 # centrally by ProcessingLLMWrapper (config/llm_config.py). Use this only on
41 # text NOT from a fresh wrapped invoke (accumulated/concatenated text, or
42 # agent/bind_tools output that bypasses the wrapper).
43 # Remove paired <think>...</think> tags
44 text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
45 # Remove any orphaned opening or closing think tags
46 text = re.sub(r"</think>", "", text)
47 text = re.sub(r"<think>", "", text)
48 return text.strip()
51# Sentinel values used by the journal reputation filter alongside the
52# numeric 1-10 quality scores. Distinguish structurally different
53# "not scored" cases so the renderer can show the user *why* the tag
54# isn't a numeric quality tier:
55#
56# - QUALITY_PENDING: reference DB hadn't finished building when the
57# search ran (first-search-during-install case).
58# - QUALITY_PREPRINT: result has no journal_ref at all (pure arxiv
59# preprint or similar); there's no venue to score. Distinct from
60# "venue unknown to our catalog" (that becomes score 3, rendered
61# as Unranked).
62QUALITY_PENDING = "pending"
63QUALITY_PREPRINT = "preprint"
66def _format_quality_tag(quality) -> str:
67 """Format a journal quality score as a compact tag for source lists.
69 The output is plaintext / Markdown. **Do NOT** render the containing
70 string through a template filter like ``{{ foo|safe }}`` or
71 ``DOMPurify.sanitize(..., {ALLOWED_TAGS:['a']})`` without first HTML-
72 escaping the surrounding title — the tag itself is safe, but a
73 downstream caller that concatenates ``title + quality_tag`` and
74 emits the result as HTML will leak any tags in ``title`` (XSS).
76 See :func:`_format_quality_tag_html` for the HTML-safe variant.
78 Accepts int | None for scored journals, plus the string sentinels
79 ``QUALITY_PENDING`` and ``QUALITY_PREPRINT``. Every numeric value
80 in VALID_QUALITY_SCORES has its own explicit branch so a bad
81 scoring-logic change can't silently rebucket a score — unexpected
82 values fall through to a debug tag that shows the raw value.
83 """
84 if quality is None:
85 return ""
86 if quality == QUALITY_PENDING:
87 return (
88 " [journal quality data is downloading in the background; "
89 "by the time you open /metrics/journals it may already "
90 "be complete — re-run this search in a minute to get "
91 "real quality scores]"
92 )
93 if quality == QUALITY_PREPRINT:
94 # No venue at all (arxiv preprint / working paper / dataset).
95 # Distinct from score 3 ("we looked and didn't find the
96 # venue") — here there's nothing *to* look up.
97 return " [preprint — not in journal catalog]"
98 # Numeric tiers. Explicit per-score branches instead of ``>=``
99 # ranges so boundary changes can't silently shift a bucket.
100 if quality == 10:
101 return " [Q1 ★★★★★]"
102 # KNOWN-DEFERRED: quality == 9 is a dead branch —
103 # constants.VALID_QUALITY_SCORES excludes 9 and the filter rejects
104 # any LLM output of that value. Kept defensively so a future change
105 # to VALID_QUALITY_SCORES does not require editing the formatter.
106 # Post-merge candidate for removal together with any score-9
107 # reintroduction work.
108 if quality == 9:
109 return " [Q1 ★★★★★]"
110 if quality == 8:
111 return " [Q1 ★★★★]"
112 if quality == 7:
113 return " [Q1 ★★★★]"
114 if quality == 6:
115 return " [Q2 ★★★]"
116 if quality == 5:
117 return " [Q2 ★★★]"
118 if quality == 4:
119 # JOURNAL_QUALITY_DEFAULT — venue found in the catalog but
120 # with no h-index / quartile / DOAJ signal.
121 return " [Unranked ★]"
122 if quality == 3:
123 # Low-confidence fallback — venue didn't match any tier. We
124 # don't know the journal, not "we know it's low-quality".
125 return " [Unranked ★]"
126 if quality == 2: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true
127 return " [Q4 ★]"
128 if quality == 1:
129 # Predatory. Usually auto-removed before this renderer sees
130 # it, but surfaces if whitelisted or the threshold is 1.
131 return " [Q4 ★]"
132 # Out-of-set value — VALID_QUALITY_SCORES gates the inputs so this
133 # is unreachable in normal operation. Show the raw value so bad
134 # data surfaces visibly instead of silently bucketing into Q4.
135 return f" [quality={quality!r}]"
138def _format_quality_tag_html(quality, *, title: str = "") -> str:
139 """HTML-safe wrapper for :func:`_format_quality_tag`.
141 Callers that render search-result titles + quality tags into an
142 HTML page must use this variant and pass the raw ``title`` so both
143 are escaped together. The quality tag itself is plaintext, but the
144 brackets and stars are safe to emit verbatim — the danger is the
145 untrusted ``title`` that a downstream HTML template may concatenate
146 alongside the tag.
148 Returns:
149 ``"{escaped_title}{quality_tag}"`` where ``escaped_title`` is
150 HTML-escaped with ``html.escape(..., quote=True)`` so quotes,
151 angle brackets, and ampersands are rendered as text.
152 """
153 import html as _html
155 return _html.escape(title, quote=True) + _format_quality_tag(quality)
158def extract_links_from_search_results(search_results: List[Dict]) -> List[Dict]:
159 """
160 Extracts links and titles from a list of search result dictionaries.
162 Each dictionary is expected to have at least the keys "title" and "link".
164 Returns a list of dictionaries with 'title' and 'url' keys.
165 """
166 links = []
167 if not search_results:
168 return links
170 for result in search_results:
171 try:
172 # Ensure we handle None values safely before calling strip()
173 title = result.get("title", "")
174 url = result.get("link", "")
175 index = result.get("index", "")
177 # Apply strip() only if the values are not None
178 title = title.strip() if title is not None else ""
179 url = url.strip() if url is not None else ""
180 index = index.strip() if index is not None else ""
182 if title and url:
183 link = {
184 "title": title,
185 "url": url,
186 "index": index,
187 "journal_quality": result.get("journal_quality"),
188 }
189 # Preserve citation-relevant fields from search engines
190 # so they reach the database (previously lost here)
191 for key in (
192 "doi",
193 "authors",
194 "published",
195 "publication_date",
196 "year",
197 "date",
198 "volume",
199 "issue",
200 "pages",
201 "journal_ref",
202 "journal",
203 "venue",
204 "publisher",
205 "source_type",
206 # Keep verified capture provenance through the live
207 # Sources rebuild, including its archived-date suffix.
208 "original_url",
209 "requested_url",
210 "timestamp",
211 "formatted_date",
212 "openalex_source_id",
213 "source",
214 "source_engine",
215 "pmid",
216 "pmcid",
217 "arxiv_id",
218 "isbn",
219 "citations",
220 "is_open_access",
221 "abstract",
222 "metadata",
223 # The collector's own page title. This rebuild drops
224 # every key not named here, so the live report path
225 # (``_format_citations`` -> here -> the renderer) lost
226 # it while a direct render kept it.
227 FETCHED_TITLE_KEY,
228 ):
229 val = result.get(key)
230 if val is not None:
231 link[key] = val
232 links.append(link)
233 except Exception:
234 # Log the specific error for debugging
235 logger.exception("Error extracting link from result")
236 continue
237 return links
240def _sanitize_sources_field(value: str) -> str:
241 """Flatten a value being rendered into the Sources block.
243 Titles come from search-result metadata — i.e. from whatever a page
244 calls itself — and were previously rendered verbatim, so a crafted
245 title could forge a whole extra numbered citation pointing anywhere.
246 URLs get the same treatment because the non-library canonical-key
247 fallback returns arbitrary scheme-less paths unchanged.
249 Control characters are replaced with a space rather than dropped, so a
250 forged ``\n[9] Fake`` degrades to visible text on the same line instead
251 of silently vanishing.
252 """
253 if not value:
254 return value
255 return "".join(
256 " " if is_line_breaking_char(ch) else ch for ch in value
257 ).strip()
260def _owned_chunk_display(recorded: object, canon: str) -> str | None:
261 """Return a recorded chunk anchor only if it names *this* citation.
263 .. note::
264 This check is the PRIMARY control, not defence in depth. Only
265 ``SearchResultsCollector`` strips a producer-supplied key at
266 ingest, and it exists solely in the LangGraph strategy —
267 ``source_based``, ``focused_iteration`` and
268 ``topic_organization`` all extend ``all_links_of_system`` with
269 raw engine dicts. So on those paths a key present here is
270 producer-supplied by construction, and the residual is bounded to
271 what this function permits: a wrong chunk or view segment of the
272 correctly-identified document, never a foreign document or an
273 arbitrary URL.
275 Shape validation alone is not enough. The value is preferred over the
276 entry's own url, so a well-formed anchor for a DIFFERENT document —
277 which arrives for free, since ``add_results`` copies the engine's dict
278 — would render and persist under this citation. The writer refuses to
279 record a foreign anchor; the reader has to refuse to read one, or the
280 two disagree about which spelling is authoritative.
282 Comparing canonical keys is exact here: ``canonical_url_key`` collapses
283 every view of a library document onto one key, so an anchor for the
284 same document matches and an anchor for any other does not.
285 """
286 if not isinstance(recorded, str):
287 return None
288 display = preferred_chunk_display(recorded)
289 if display is None: 289 ↛ 290line 289 didn't jump to line 290 because the condition on line 289 was never true
290 return None
291 return display if canonical_url_key(display) == canon else None
294def source_url_field(link: Dict) -> object:
295 """The field a source's identity is read from: ``link`` first.
297 ``SearchResultsCollector`` keys citations on
298 ``_citation_dedup_key(result["link"])``, and records the authoritative
299 chunk anchor or normalized URL in ``link``. A result carrying BOTH
300 fields with divergent values (e.g. ``link`` carrying the collector's
301 rebuilt anchor while ``url`` retains an anchor-less engine string) must
302 consistently read ``link`` so the entry groups under its intended chunk
303 anchor rather than fanning out into an unintended second group.
305 ``link`` wins because it is what the collector keys on, what
306 ``_format_results`` shows the agent next to the ``[N]`` marker, and
307 what ``find_by_url`` resolves — i.e. it is what the citation index
308 actually means. ``url`` remains the fallback for the raw engine dicts
309 the non-LangGraph strategies extend ``all_links_of_system`` with,
310 some of which set only that key.
311 """
312 return link.get("link") or link.get("url") or ""
315# ---------------------------------------------------------------------------
316# Uncited-source filter engine (#5379).
317#
318# Heuristic parser: extracts citation signals (numeric bracket indices,
319# tagged citations, "Source N" phrases, inline URLs) from report prose so
320# the final ``## Sources`` bibliography can drop accumulated search hits
321# that were never cited. Over-inclusion is safe (extra bibliography line);
322# under-inclusion silently drops a cited source, so every ambiguous or
323# indeterminate case falls back to the full list in ``fallback`` mode.
324# ---------------------------------------------------------------------------
326_MAX_RANGE_SPAN = 1000
327_MAX_TOTAL_EXPANSION = 5000
328_MAX_RANGE_ENDPOINT_DIGITS = 18
329# This bounds candidate expansion work, not the report or its context. On
330# exhaustion the parser becomes indeterminate and keeps the full bibliography.
331_MAX_URL_CANDIDATE_CHARS = 1_000_000
334class _URLCandidateBudgetExceeded(Exception):
335 """The citation parser must fall back before expanding more candidates."""
338class _URLCandidateBudget:
339 def __init__(self):
340 self.remaining = _MAX_URL_CANDIDATE_CHARS
342 def consume(self, characters: int) -> None:
343 if characters > self.remaining:
344 raise _URLCandidateBudgetExceeded
345 self.remaining -= characters
348_FENCE_MARKERS = ("```", "~~~")
350# Ordinary prose tokens that must NOT force the indeterminate fallback:
351# ordinals ("1st", "2nd", "3rd"), units with g-z suffixes ("5x", "100m"),
352# multi-letter suffixes ("12px"), percentages ("50%"). Single bracket
353# tokens of digits immediately followed by such suffixes are prose, not
354# attempted citations. Only contents with list/range syntax (",", "-",
355# "–", "—", "and", "&") look like attempted citation syntax and may
356# trigger indeterminate.
357#
358# Single-letter a-f suffixes ("3a", "4b", "12c") are DELIBERATELY excluded:
359# they are regex-identical to sub-reference citations, so they stay on the
360# safe (indeterminate → fallback) side instead of being silently ignored.
361_PROSE_TOKEN_RE = re.compile(r"\d+[A-Za-z%]+")
362_SUBREF_SINGLE_LETTERS = frozenset("abcdefABCDEF")
363_LIST_SYNTAX_RE = re.compile(r",|[\-–—]|&|\band\b", re.IGNORECASE)
365# Single/double ASCII + CJK full-width brackets with numeric content:
366# [N], [N, M], [[N]], 【N】, [N-M], [N–M], [ 1, 2 ] (outer spaces allowed).
367# Letters excluded so [GPT-4]/[COVID-19] never match here
368# (handled/rejected as tagged).
369_BRACKET_NUMERIC_RE = re.compile(
370 r"[\[【]\[?\^?\s*([0-9][0-9\s,\-–—]*[0-9]|[0-9])\s*\]?[\]】]"
371)
373# Tagged citations: [arxiv-1], [arxiv.org-2], [openai.com-2]. Tag must
374# contain a letter or dot so pure ranges ([1-3]) never match as tag "1".
375_TAGGED_CITATION_RE = re.compile(r"\[(?P<tag>[A-Za-z0-9._/\-]+)-(?P<idx>\d+)\]")
377# "Source 1", "Sources 1 and 2", "sources 1, 2 & 3", "Sources 1-3"
378# (case-insensitive). Dash/en-dash/em-dash ranges are part of the capture
379# so "Sources 1-3" parses the whole range instead of just the "1" prefix
380# (which previously kept only source 1 with no fallback).
381_SOURCE_PHRASE_RE = re.compile(
382 r"(?i)\bsources?\s+(?P<nums>\d+(?:\s*[\-–—]\s*\d+)?"
383 r"(?:\s*(?:,|and|&)\s*\d+(?:\s*[\-–—]\s*\d+)?)*)"
384)
386# Broad capture; paren-balance + trailing-punct peeling happens per match.
387# Bracketed IP-literal authorities need their own branch: brackets delimit
388# the host here, while brackets elsewhere still delimit prose citations.
389_IP_LITERAL_AUTHORITY = (
390 r"https?://(?:[^\s<>\[\]\"/?#@]*@)?\[[^\s<>\[\]\"/?#@]+\]"
391)
392_IP_LITERAL_URL_RE = re.compile(_IP_LITERAL_AUTHORITY, re.IGNORECASE)
394# Replay targets can contain IP authorities and bracketed path/query values.
395# A replay-specific scanner retains nested pairs while unmatched closing
396# brackets still delimit surrounding prose, as they do for ordinary URLs.
397_WAYBACK_REPLAY_PREFIX = (
398 r"https?://web\.archive\.org(?::(?:80|443))?/web/"
399 r"[0-9]{14}(?:id_|if_)?/https?://"
400)
401_WAYBACK_REPLAY_PREFIX_RE = re.compile(_WAYBACK_REPLAY_PREFIX, re.IGNORECASE)
402_WAYBACK_REPLAY_URL = _WAYBACK_REPLAY_PREFIX + r'[^\s<>"]+'
404# In addition to absolute URLs and markdown-link destinations
405# (``](/library/…)``), bare library routes (``/library/document/…``,
406# ``/lib/document/…``) are in scope by design: prose commonly mentions
407# them outside link syntax, and dropping a bare-path mention is the same
408# citation-loss class as dropping a markdown destination.
409_ORDINARY_ABSOLUTE_URL = (
410 rf"{_IP_LITERAL_AUTHORITY}[^\s<>\[\]\"]*"
411 r"|https?://[^\s<>\[\]\"]+"
412)
413_RELATIVE_CITATION_URL = (
414 r"|\]\([ \t]*(?P<relative>/[^\s<>\"]*)"
415 r"|(?P<bare_library>/(?:library|lib)/document/[^\s<>\[\]\"]+)"
416)
417_URL_RE = re.compile(
418 rf"(?P<absolute>{_WAYBACK_REPLAY_URL}|{_ORDINARY_ABSOLUTE_URL})"
419 + _RELATIVE_CITATION_URL,
420 re.IGNORECASE,
421)
422# Find a replay prefix before scanning its target. Capturing the whole
423# remaining suffix here would rescan adjacent wrapped links quadratically.
424_URL_START_RE = re.compile(
425 rf"(?P<absolute>{_WAYBACK_REPLAY_PREFIX}|{_ORDINARY_ABSOLUTE_URL})"
426 + _RELATIVE_CITATION_URL,
427 re.IGNORECASE,
428)
430# ``canonical_url_key`` lowercases scheme/host but keeps path case, and the
431# library-route parser matches ``/library/document/`` case-sensitively
432# (like the Flask route and ``library_resolver``). The prose regex above
433# captures ``/LIBRARY/Document/…`` case variants, so normalize just the
434# route prefix to lowercase before keying — the doc id itself keeps its
435# case. Without this the mention is captured but never matches its link.
436_BARE_LIBRARY_PREFIX_RE = re.compile(
437 r"^/(?:library|lib)/document/", re.IGNORECASE
438)
441def _normalize_bare_library_prefix(candidate: str) -> str:
442 """Lowercase only the ``/library|lib/document/`` route prefix."""
443 return _BARE_LIBRARY_PREFIX_RE.sub("/library/document/", candidate)
446# Sentence-final + CJK punctuation stripped from prose URLs. Deliberately
447# excludes closing delimiters ( ) ] } > ' " ` ) — those are balanced
448# separately so a balanced ")" before a final "." survives.
449_URL_SENTENCE_PUNCT = ".,;:!?。、，！？；：「」『』、…·—–"
450_URL_WRAPPER_PUNCT = "'\"`]}>】”’»》“‘«›"
451_URL_WRAPPERS = {
452 "(": ")",
453 "[": "]",
454 "{": "}",
455 "【": "】",
456 "「": "」",
457 "『": "』",
458 "《": "》",
459 "〈": "〉",
460 "“": "”",
461 "„": "“",
462 "‘": "’",
463 "‚": "‘",
464 "«": "»",
465 "»": "«",
466 "‹": "›",
467 "'": "'",
468 '"': '"',
469 "`": "`",
470 "*": "*",
471 "_": "_",
472}
473_MARKDOWN_URL_END_RE = re.compile(
474 r"[ \t\r\n]*(?:(?:\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'"
475 r"|\((?:\\.|[^()\\])*\))[ \t\r\n]*)?\)"
476)
477_MARKDOWN_URL_ESCAPE_RE = re.compile(r"\\([!-/:-@\[-`{-~])")
479UNCITED_SOURCES_MODE_KEY = "report.uncited_sources_mode"
480UNCITED_SOURCES_MODES = ("fallback", "strict", "disabled")
481UNCITED_SOURCES_DEFAULT = "fallback"
484def resolve_uncited_sources_mode(raw=None, settings_snapshot=None) -> str:
485 """Normalize the ``report.uncited_sources_mode`` setting.
487 ``raw`` is an explicitly provided value (e.g. from a strategy's
488 settings snapshot, already unwrapped). ``None`` resolves from
489 ``settings_snapshot`` (when given) or the thread settings context,
490 defaulting to ``"fallback"`` when no context is bound. Unknown values
491 degrade to ``"fallback"`` so a stale or hand-edited setting can never
492 wipe out a bibliography. Settings lookup failures also degrade to the
493 default: rendering a report must never crash on a preference read.
494 """
495 if raw is None:
496 try:
497 from ..config.thread_settings import get_setting_from_snapshot
499 raw = get_setting_from_snapshot(
500 UNCITED_SOURCES_MODE_KEY,
501 UNCITED_SOURCES_DEFAULT,
502 settings_snapshot=settings_snapshot,
503 )
504 except Exception:
505 return UNCITED_SOURCES_DEFAULT
506 if isinstance(raw, dict): 506 ↛ 507line 506 didn't jump to line 507 because the condition on line 506 was never true
507 raw = raw.get("value", UNCITED_SOURCES_DEFAULT)
508 if isinstance(raw, str) and raw.strip() in UNCITED_SOURCES_MODES: 508 ↛ 510line 508 didn't jump to line 510 because the condition on line 508 was always true
509 return raw.strip()
510 return UNCITED_SOURCES_DEFAULT
513def _strip_fence_prefix(s: str) -> str:
514 """Strip Markdown quote/list prefixes from a line-start string.
516 Handles ``>``, ``-``/``*``/``+``, ``1.``/``1)`` nesting (e.g.
517 ``> - ```js``). Capped at 32 iterations so a hostile single line
518 of ``\"1. \"`` repetitions cannot turn the per-line slice into a
519 quadratic scan; realistic nesting is single digits.
520 """
521 for _ in range(32): 521 ↛ 536line 521 didn't jump to line 536 because the loop on line 521 didn't complete
522 if s.startswith(">"):
523 s = s[1:].lstrip(" \t")
524 continue
525 if len(s) >= 2 and s[0] in "-*+" and s[1] in " \t":
526 i = 1
527 while i < len(s) and s[i] in " \t":
528 i += 1
529 s = s[i:]
530 continue
531 m = re.match(r"\d+[.)][ \t]+", s)
532 if m:
533 s = s[m.end() :]
534 continue
535 break
536 return s
539def _is_fence_line(line: str, marker: str) -> bool:
540 """True when ``line`` can *open* a ``marker`` code block.
542 A fence is recognized at the start of a line after optional
543 spaces/tabs indent *and* optional Markdown quote/list prefixes
544 (``>``, ``-``/``*``/``+``, ``1.``/``1)`` — e.g. ``> ```json`` or
545 ``- ```js``). Those shapes are how models quote tool output and
546 search results; ignoring the opener while counting a later plain
547 ````` closer pairs the closer with the next real opener and strips
548 real citations between them (source loss in the default mode).
550 A line whose stripped remainder starts with ``marker`` but contains
551 a second ``marker`` *after the initial fence run* (````` decorate
552 [8] ``` end``) is a decorative single-line run, not an opener, and
553 returns False so it never opens a block. A longer single run
554 (`` `````` ``) has no second marker after its run and still opens.
555 Closers are deliberately not vetted here — see
556 :func:`_is_fence_closer_line`.
557 """
558 content = line.rstrip("\r\n")
559 s = _strip_fence_prefix(content.lstrip(" \t"))
560 if not s.startswith(marker):
561 return False
562 # Initial fence run (3+ backticks/tildes): a longer single run is
563 # one fence, not two.
564 fence_char = marker[0]
565 run = 0
566 while run < len(s) and s[run] == fence_char:
567 run += 1
568 # Second marker after the run → decorative single-line run.
569 if marker in s[run:]:
570 return False
571 return True
574def _is_fence_closer_line(line: str, marker: str) -> bool:
575 """True when ``line`` can *close* an open ``marker`` code block.
577 Same prefix rule as the opener, but with no double-marker veto: a
578 closer carrying trailing text (````` done ``` ``) or a longer fence
579 (`` `````` `` closing a `` ``` `` block) must still close. Vetoing
580 it here pairs the opener with the *next* opener and strips real
581 prose between them with no indeterminate fallback.
582 """
583 content = line.rstrip("\r\n")
584 s = _strip_fence_prefix(content.lstrip(" \t"))
585 return s.startswith(marker)
588def _find_fence_positions(prose: str, marker: str) -> list[int]:
589 """Return line-start offsets of ``marker`` fence *openers*.
591 Recognizes line-anchored markers plus blockquoted/bulleted forms
592 (``> ```json``, ``- ```js``) via :func:`_is_fence_line`; inline
593 decorative runs and single-line double-marker runs are not openers.
594 Closers (which carry no veto) are paired statefully in
595 :func:`_strip_marker_blocks`, not by blind consecutive pairing.
596 """
597 positions: list[int] = []
598 offset = 0
599 for line in prose.splitlines(keepends=True):
600 if _is_fence_line(line, marker):
601 positions.append(offset)
602 offset += len(line)
603 return positions
606def _strip_marker_blocks(prose: str, marker: str) -> tuple[str, bool]:
607 """Strip paired ``marker`` blocks; True dangling if odd.
609 Stateful pairing: the double-marker veto applies only to *openers*
610 (so ````` decorate [8] ``` end`` never opens), while any
611 line-anchored ``marker`` *closes* an open block (so ````` done ```
612 `` and six-backtick `` `````` `` closers still close instead of
613 letting the opener pair across real prose). Full opener-through-
614 closer lines are excluded; an unterminated block returns the
615 original prose with dangling True so the caller falls back to the
616 full bibliography.
617 """
618 lines = prose.splitlines(keepends=True)
619 if not lines: 619 ↛ 620line 619 didn't jump to line 620 because the condition on line 619 was never true
620 return prose, False
621 spans: list[tuple[int, int]] = []
622 in_block = False
623 block_start = 0
624 for i, line in enumerate(lines):
625 if not in_block:
626 if _is_fence_line(line, marker):
627 in_block = True
628 block_start = i
629 else:
630 if _is_fence_closer_line(line, marker):
631 in_block = False
632 spans.append((block_start, i))
633 if in_block:
634 return prose, True
635 if not spans:
636 return prose, False
637 exclude: set[int] = set()
638 for span_start, span_end in spans:
639 for i in range(span_start, span_end + 1):
640 exclude.add(i)
641 return "".join(
642 line for i, line in enumerate(lines) if i not in exclude
643 ), False
646def _strip_fenced_code(prose: str) -> tuple[str, bool]:
647 """Strip fenced code blocks (``` and ~~~); dangling either → indeterminate.
649 Fence lines include blockquoted/bulleted forms (``> ```json``,
650 ``- ```js``) so a quoted opener pairs with its closer instead of
651 letting a later plain marker pair across real prose. Inline
652 decorative runs (``~~~ config [2] ~~~``) and single-line
653 double-marker runs (```` ``` decorate [8] ``` end``) never *open* a
654 block, but any line-anchored marker still *closes* an open block
655 (so ````` done ``` `` and six-backtick closers close rather than
656 pairing the opener across real prose). Each marker type pairs only
657 with itself so a ``` block never swallows across a ~~~ fence. A
658 dangling (unterminated) marker of either type marks the parse
659 indeterminate instead of stripping half the report (a single stray
660 fence in quoted web content would otherwise swallow real citations
661 between it and the next block).
662 """
663 indeterminate = False
664 text = prose
665 for marker in _FENCE_MARKERS:
666 text, dangling = _strip_marker_blocks(text, marker)
667 if dangling:
668 indeterminate = True
669 return text, indeterminate
672def _is_all_uppercase_tag(tag: str) -> bool:
673 letters = [ch for ch in tag if ch.isalpha()]
674 return bool(letters) and all(ch.isupper() for ch in letters)
677def _is_ignorable_prose_singleton(content: str) -> bool:
678 """True when a digit-led singleton is ordinary prose, not a sub-ref."""
679 if "%" in content:
680 return True
681 m = re.fullmatch(r"\d+([A-Za-z]+)", content)
682 if m is None: 682 ↛ 683line 682 didn't jump to line 683 because the condition on line 682 was never true
683 return True
684 suffix = m.group(1)
685 if len(suffix) >= 2:
686 return True
687 return suffix not in _SUBREF_SINGLE_LETTERS
690def _is_malformed_numeric_content(content: str) -> bool:
691 """True when a digit-led bracket looks like a broken citation list.
693 Ordinary prose singletons (ordinals ``2nd``/``1st``, ``g``-``z`` units
694 ``5x``/``100m``, multi-letter ``12px``, percentages ``50%``) are
695 explicitly NOT malformed: they are digits immediately followed by a
696 safe suffix with no list/range syntax, so they are ignored instead of
697 forcing the full-bibliography fallback (which would defeat ``strict``
698 mode's cited-only contract). Single-letter ``a``-``f`` suffixes
699 (``3a``/``4b``/``12c``) are sub-reference-shaped and stay indeterminate
700 so fallback preserves the bibliography. Only contents containing
701 list/range syntax (``,``, ``-``/``–``/``—``, ``and``/``&``) look like
702 attempted citation syntax; a bad chunk there
703 (``[1,abc,3]``, ``[1-abc]``) stays indeterminate so fallback preserves
704 the bibliography. Other singletons without list syntax (``[1 abc]``,
705 ``[1 and 2]``) also stay indeterminate — ignoring them would risk
706 silently dropping an intended citation.
707 """
708 content = content.strip()
709 if not content or not content[0].isascii() or not content[0].isdigit():
710 return False
711 if not _LIST_SYNTAX_RE.search(content):
712 # Single token, no list/range syntax. Prose shapes (digits +
713 # letters/% suffix) are ignored; anything else falls through to
714 # the chunk check below (e.g. "1 abc" has a space so it does not
715 # match _PROSE_TOKEN_RE and is still indeterminate).
716 if _PROSE_TOKEN_RE.fullmatch(content):
717 if _is_ignorable_prose_singleton(content):
718 return False
719 for chunk in content.split(","):
720 chunk = chunk.strip()
721 if not chunk: 721 ↛ 722line 721 didn't jump to line 722 because the condition on line 721 was never true
722 return True
723 if chunk.isascii() and chunk.isdigit():
724 continue
725 if re.fullmatch(r"[0-9]+\s*[\-–—]\s*[0-9]+", chunk):
726 continue
727 return True
728 return False
731def _peel_url_candidate(raw: str) -> str:
732 """Peel punctuation and unmatched delimiters from an ambiguous prose URL.
734 Sentence punctuation is peeled BEFORE balancing delimiters: a balanced
735 Wikipedia-style ``...(lang).`` first loses the final ``.`` leaving a
736 balanced ``...(lang)`` which is kept. The old order stripped ``.`` and
737 ``)`` together via a single ``rstrip``, turning the balanced URL into
738 the unbalanced ``...(lang`` and dropping the source.
739 """
740 url = raw
741 authority_match = _IP_LITERAL_URL_RE.match(url)
742 # Parentheses in credentials or an IPvFuture host belong to the
743 # authority, so they cannot balance path or prose-wrapper parentheses.
744 suffix = url[authority_match.end() :] if authority_match else url
745 unmatched_closing = max(0, suffix.count(")") - suffix.count("("))
746 unmatched_brackets = max(0, url.count("]") - url.count("["))
747 end = len(url)
748 while end: 748 ↛ 763line 748 didn't jump to line 763 because the condition on line 748 was always true
749 trailing = url[end - 1]
750 if trailing == "]" and not unmatched_brackets: 750 ↛ 752line 750 didn't jump to line 752 because the condition on line 750 was never true
751 # A balanced closing bracket belongs to an IP-literal host.
752 break
753 if trailing in _URL_SENTENCE_PUNCT or trailing in _URL_WRAPPER_PUNCT:
754 if trailing == "]": 754 ↛ 755line 754 didn't jump to line 755 because the condition on line 754 was never true
755 unmatched_brackets -= 1
756 end -= 1
757 continue
758 if trailing == ")" and unmatched_closing:
759 unmatched_closing -= 1
760 end -= 1
761 continue
762 break
763 return url[:end]
766def _url_citation_candidates(
767 match: re.Match, budget: Optional[_URLCandidateBudget] = None
768) -> Set[str]:
769 """Preserve exact destinations; keep ambiguous bare URL interpretations."""
770 if budget is None: 770 ↛ 771line 770 didn't jump to line 771 because the condition on line 770 was never true
771 budget = _URLCandidateBudget()
772 group = next(
773 name
774 for name in ("absolute", "relative", "bare_library")
775 if match.group(name) is not None
776 )
777 raw = match.group(group)
778 budget.consume(len(raw))
779 start = match.start(group)
780 text = match.string
781 following = text[match.end() : match.end() + 1]
782 if start and text[start - 1] == "<" and following == ">":
783 # Angle brackets delimit URL content, including punctuation.
784 return {_MARKDOWN_URL_ESCAPE_RE.sub(r"\1", raw)}
785 if start and text[start - 1] == '"' and following == '"':
786 # Quotes delimit the URL, but sentence punctuation inside them may
787 # belong to the surrounding sentence; keep both readings.
788 budget.consume(len(raw))
789 return {raw, _peel_url_candidate(raw)}
791 prefix = start
792 while prefix and text[prefix - 1] in " \t\r\n":
793 prefix -= 1
794 if text[max(0, prefix - 2) : prefix] == "](":
795 depth = 0
796 escaped = False
797 for position, char in enumerate(raw):
798 if escaped: 798 ↛ 799line 798 didn't jump to line 799 because the condition on line 798 was never true
799 escaped = False
800 elif char == "\\": 800 ↛ 801line 800 didn't jump to line 801 because the condition on line 800 was never true
801 escaped = True
802 elif char == "(":
803 depth += 1
804 elif char == ")":
805 if not depth:
806 budget.consume(position)
807 return {_MARKDOWN_URL_ESCAPE_RE.sub(r"\1", raw[:position])}
808 depth -= 1
809 if not depth and _MARKDOWN_URL_END_RE.match(text, match.end()): 809 ↛ 814line 809 didn't jump to line 814 because the condition on line 809 was always true
810 return {_MARKDOWN_URL_ESCAPE_RE.sub(r"\1", raw)}
812 # Remove one sentence mark outside any surrounding wrappers. Further
813 # punctuation, underscores and stars may be significant URL content.
814 if raw and raw[-1] in _URL_SENTENCE_PUNCT:
815 budget.consume(len(raw) - 1)
816 url = raw[:-1]
817 else:
818 url = raw
819 wrapped = False
820 for _ in range(32): 820 ↛ 842line 820 didn't jump to line 842 because the loop on line 820 didn't complete
821 if not start:
822 break
823 closer = _URL_WRAPPERS.get(text[start - 1])
824 if not closer or not url.endswith(closer):
825 break
826 authority = _IP_LITERAL_URL_RE.match(url)
827 if closer == "]" and authority and authority.end() == len(url): 827 ↛ 829line 827 didn't jump to line 829 because the condition on line 827 was never true
828 # The captured bracket closes the host, not an outer wrapper.
829 break
830 if (
831 closer in "])"
832 and _WAYBACK_REPLAY_PREFIX_RE.match(url)
833 and url.count(text[start - 1]) == url.count(closer)
834 ):
835 # The replay branch excludes an unmatched outer closer, so a
836 # balanced final bracket/parenthesis belongs to the target.
837 break
838 budget.consume(len(url) - len(closer))
839 url = url[: -len(closer)]
840 start -= 1
841 wrapped = True
842 if wrapped:
843 # The wrapper closer is prose, but any sentence mark it hides may
844 # be URL content or sentence punctuation; keep both readings so
845 # ``"https://x.example/a."`` cites ``a`` and ``a.`` alike.
846 budget.consume(len(url))
847 return {url, url.rstrip(_URL_SENTENCE_PUNCT)}
849 candidates = {raw, url}
850 if _WAYBACK_REPLAY_PREFIX_RE.match(url):
851 # A bare replay followed immediately by [N] can spell either a
852 # target suffix or a prose reference. Keep both readings; exact
853 # angle/Markdown destinations already returned above.
854 # Find the final contiguous run without retaining a match object
855 # for every bracket. Charge each interpretation BEFORE slicing it:
856 # preserving all prefixes as strings otherwise takes quadratic space.
857 run_start = None
858 previous_end = -1
859 for reference in _BRACKET_NUMERIC_RE.finditer(url):
860 if reference.start() != previous_end:
861 run_start = reference.start()
862 previous_end = reference.end()
863 if previous_end == len(url) and run_start is not None:
864 for reference in _BRACKET_NUMERIC_RE.finditer(url, run_start):
865 end = reference.start()
866 budget.consume(end)
867 candidates.add(url[:end])
868 # A run of three or more trailing sentence marks reads as an
869 # ellipsis-sized prose unit (``See https://x.example/a... Also``), so
870 # the fully-peeled reading stays plausible too. Shorter runs peel
871 # exactly one mark: a sentence contributes a single terminator, and
872 # peeling deeper would strip real URL content (``Read
873 # https://x.example/a..`` where the path genuinely ends in ``.``).
874 peeled_end = len(raw)
875 while peeled_end and raw[peeled_end - 1] in _URL_SENTENCE_PUNCT:
876 peeled_end -= 1
877 if len(raw) - peeled_end >= 3: 877 ↛ 878line 877 didn't jump to line 878 because the condition on line 877 was never true
878 budget.consume(peeled_end)
879 candidates.add(raw[:peeled_end])
880 authority = _IP_LITERAL_URL_RE.match(url)
881 suffix = url[authority.end() :] if authority else url
882 if suffix.count(")") > suffix.count("("):
883 # A closing parenthesis can be URL content or belong to prose
884 # whose opening parenthesis precedes intervening words.
885 budget.consume(len(raw))
886 candidates.add(_peel_url_candidate(raw))
887 return candidates
890def _has_malformed_numeric_brackets(text: str) -> bool:
891 """Check whole bracket contents in one pass, including long/multiline ones."""
892 start: Optional[int] = None
893 for position, char in enumerate(text):
894 if char in "[【":
895 if start is None or position == start:
896 start = position + 1
897 elif char in "]】" and start is not None:
898 content = text[start:position].lstrip()
899 if content.startswith("^"):
900 content = content[1:]
901 if _is_malformed_numeric_content(content):
902 return True
903 start = None
904 return False
907def _mask_replay_ip_literal(match: re.Match) -> str:
908 """Mask only a parseable IP authority inside a verified replay target."""
909 authority = match.group(0)
910 try:
911 _ = urlsplit(authority).port
912 except ValueError:
913 return authority
914 return authority.replace("[", "(").replace("]", ")")
917def _citation_url_matches(text: str) -> List[re.Match]:
918 """Scan replay targets once, retaining balanced URL brackets/wrappers."""
919 matches: List[re.Match] = []
920 position = 0
921 while (match := _URL_START_RE.search(text, position)) is not None:
922 raw = match.group("absolute")
923 if raw and _WAYBACK_REPLAY_PREFIX_RE.match(raw):
924 prefix = match.start("absolute")
925 while prefix and text[prefix - 1] in " \t\r\n":
926 prefix -= 1
927 parenthesized = bool(prefix and text[prefix - 1] == "(")
928 bracket_depth = 0
929 paren_depth = 0
930 escaped = False
931 end = match.end()
932 while end < len(text):
933 character = text[end]
934 if character.isspace() or character in '<>"':
935 break
936 if escaped: 936 ↛ 937line 936 didn't jump to line 937 because the condition on line 936 was never true
937 escaped = False
938 elif parenthesized and character == "\\": 938 ↛ 939line 938 didn't jump to line 939 because the condition on line 938 was never true
939 escaped = True
940 elif character == "[":
941 bracket_depth += 1
942 elif character == "]":
943 if not bracket_depth:
944 break
945 bracket_depth -= 1
946 elif parenthesized and character == "(":
947 paren_depth += 1
948 elif parenthesized and character == ")":
949 if not paren_depth:
950 break
951 paren_depth -= 1
952 end += 1
953 bounded = _URL_RE.match(text, match.start(), end)
954 if bounded is not None: 954 ↛ 956line 954 didn't jump to line 956 because the condition on line 954 was always true
955 match = bounded
956 matches.append(match)
957 position = match.end()
958 return matches
961def _mask_ip_literal_authority(
962 match: re.Match, budget: Optional[_URLCandidateBudget] = None
963) -> str:
964 """Keep valid captured URL hosts out of the numeric citation parser."""
965 url = match.group("absolute")
966 if not url:
967 return match.group(0)
968 authority_match = _IP_LITERAL_URL_RE.match(url)
969 if authority_match is None:
970 if _WAYBACK_REPLAY_PREFIX_RE.match(url):
971 for candidate in _url_citation_candidates(match, budget): 971 ↛ 983line 971 didn't jump to line 983 because the loop on line 971 didn't complete
972 try:
973 if not _is_wayback_replay_url( 973 ↛ 976line 973 didn't jump to line 976 because the condition on line 973 was never true
974 candidate, urlsplit(candidate)
975 ):
976 continue
977 except ValueError:
978 continue
979 # Only IP-host brackets are masked. Path/query brackets
980 # and surrounding numeric or malformed references remain
981 # visible to the normal citation parser.
982 return _IP_LITERAL_URL_RE.sub(_mask_replay_ip_literal, url)
983 return url
984 for candidate in _url_citation_candidates(match, budget):
985 if _IP_LITERAL_URL_RE.match(candidate) is None:
986 # A malformed Markdown destination can end before its host.
987 continue
988 try:
989 # Accessing port rejects malformed or out-of-range ports.
990 _ = urlsplit(candidate).port
991 except ValueError:
992 continue
993 break
994 else:
995 # An invalid authority must not hide an ambiguous citation bracket.
996 return url
997 authority = authority_match.group(0)
998 # Retain URL text so masking cannot join citation phrases or ranges
999 # across an authority. Non-whitespace delimiters also keep a zone ID
1000 # ending in "Sources" separate from any number following the URL.
1001 masked_authority = authority.replace("[", "(").replace("]", ")")
1002 return masked_authority + url[authority_match.end() :]
1005def _parse_bracket_content(
1006 content: str, indices: Set[int], state: dict
1007) -> None:
1008 """Expand comma lists + dash/en-dash ranges into ``indices``.
1010 Sets ``state["indeterminate"]`` when any range exceeds span, digit,
1011 or aggregate budgets instead of keeping endpoints-only (which would
1012 suppress the safety-net fallback with a partial result).
1013 """
1014 for chunk in content.split(","):
1015 chunk = chunk.strip()
1016 if not chunk: 1016 ↛ 1017line 1016 didn't jump to line 1017 because the condition on line 1016 was never true
1017 continue
1018 m = re.fullmatch(r"(\d+)\s*[\-–—]\s*(\d+)", chunk)
1019 if m:
1020 lo_s, hi_s = m.group(1), m.group(2)
1021 if (
1022 len(lo_s) > _MAX_RANGE_ENDPOINT_DIGITS
1023 or len(hi_s) > _MAX_RANGE_ENDPOINT_DIGITS
1024 ):
1025 state["indeterminate"] = True
1026 continue
1027 try:
1028 lo, hi = int(lo_s), int(hi_s)
1029 except ValueError:
1030 state["indeterminate"] = True
1031 continue
1032 if lo > hi: 1032 ↛ 1033line 1032 didn't jump to line 1033 because the condition on line 1032 was never true
1033 lo, hi = hi, lo
1034 if hi - lo > _MAX_RANGE_SPAN:
1035 state["indeterminate"] = True
1036 continue
1037 if state["expanded"] + (hi - lo + 1) > _MAX_TOTAL_EXPANSION: 1037 ↛ 1038line 1037 didn't jump to line 1038 because the condition on line 1037 was never true
1038 state["indeterminate"] = True
1039 continue
1040 for i in range(lo, hi + 1):
1041 indices.add(i)
1042 state["expanded"] += hi - lo + 1
1043 elif chunk.isascii() and chunk.isdigit(): 1043 ↛ 1052line 1043 didn't jump to line 1052 because the condition on line 1043 was always true
1044 if len(chunk) > _MAX_RANGE_ENDPOINT_DIGITS: 1044 ↛ 1045line 1044 didn't jump to line 1045 because the condition on line 1044 was never true
1045 state["indeterminate"] = True
1046 continue
1047 try:
1048 indices.add(int(chunk))
1049 except ValueError:
1050 state["indeterminate"] = True
1051 else:
1052 continue
1055def extract_cited_indices_and_urls(
1056 prose: str,
1057) -> Tuple[Set[int], Set[str], bool]:
1058 """Parse citation signals from report body prose.
1060 Returns ``(cited_indices, cited_url_keys, indeterminate)`` where
1061 ``cited_url_keys`` are ``canonical_url_key`` values. ``indeterminate``
1062 is True when a range, bracket, or fence is ambiguous or the shared URL
1063 candidate-work budget is exhausted. The result must then be treated as
1064 unreliable (caller falls back to the full list without truncating prose).
1066 Recognized signals: numeric brackets (outer spaces allowed:
1067 ``[ 1, 2 ]``), tagged citations, ``Source N`` phrases, absolute URLs,
1068 markdown-link library destinations (``](/library/…)``) AND bare
1069 library routes (``/library/document/…``, ``/lib/document/…``) mentioned
1070 outside link syntax. Route-prefix case is normalized for both bare
1071 mentions and markdown destinations, so ``/LIBRARY/Document/…``
1072 retains like the lowercase form. Fenced code (``` and ~~~,
1073 including blockquoted/bulleted forms like ``> ```json``) is stripped;
1074 inline runs and single-line double-marker runs never swallow
1075 citations; a dangling fence of either type marks the parse
1076 indeterminate.
1078 Ordinary prose singletons (ordinals ``2nd``, ``g``-``z`` units
1079 ``5x``/``100m``, multi-letter ``12px``, percentages ``50%``) are
1080 ignored — they never set indeterminate — so normal writing does not
1081 force the full-bibliography fallback in ``strict`` mode.
1082 Single-letter ``a``-``f`` suffixes (``3a``) stay indeterminate as
1083 sub-reference-shaped. Only digit-led brackets with list/range syntax
1084 that fail to parse (``[1,abc,3]``) are indeterminate.
1086 Heuristic: over-includes on years/array indices; over-inclusion is
1087 harmless because the fallback keeps extra lines, while
1088 under-inclusion drops cited sources.
1089 """
1090 indices: Set[int] = set()
1091 url_keys: Set[str] = set()
1092 state = {"indeterminate": False, "expanded": 0}
1093 candidate_budget = _URLCandidateBudget()
1094 if not prose or not prose.strip(): 1094 ↛ 1095line 1094 didn't jump to line 1095 because the condition on line 1094 was never true
1095 return indices, url_keys, False
1097 text, fence_indeterminate = _strip_fenced_code(prose)
1098 if fence_indeterminate:
1099 state["indeterminate"] = True
1101 # Keep the original text for URL extraction. A digit-led IPv6 address
1102 # must not look like a broken [N] citation and force all sources back in.
1103 # Use the same capture boundaries as URL extraction below. An IPv6
1104 # literal inside another URL's path or query is not a separate host.
1105 url_matches = _citation_url_matches(text)
1106 citation_parts: List[str] = []
1107 position = 0
1108 try:
1109 for match in url_matches:
1110 citation_parts.append(text[position : match.start()])
1111 citation_parts.append(
1112 _mask_ip_literal_authority(match, candidate_budget)
1113 )
1114 position = match.end()
1115 except _URLCandidateBudgetExceeded:
1116 return indices, url_keys, True
1117 citation_parts.append(text[position:])
1118 citation_text = "".join(citation_parts)
1120 for m in _BRACKET_NUMERIC_RE.finditer(citation_text):
1121 _parse_bracket_content(m.group(1), indices, state)
1123 if _has_malformed_numeric_brackets(citation_text):
1124 state["indeterminate"] = True
1126 for m in _TAGGED_CITATION_RE.finditer(citation_text):
1127 tag, idx_s = m.group("tag"), m.group("idx")
1128 # Check this after matching instead of using overlapping greedy
1129 # groups: those take quadratic time on long malformed brackets.
1130 if not re.search(r"[A-Za-z.]", tag):
1131 continue
1132 if _is_all_uppercase_tag(tag):
1133 continue
1134 if len(idx_s) > _MAX_RANGE_ENDPOINT_DIGITS:
1135 # Implausibly large tag index (no bibliography has 10^18
1136 # entries): clearly prose/hostile, not a citation. Ignore
1137 # instead of marking indeterminate so a single token like
1138 # [a-999…] cannot force the fallback for every reader.
1139 continue
1140 try:
1141 indices.add(int(idx_s))
1142 except ValueError:
1143 state["indeterminate"] = True
1145 for m in _SOURCE_PHRASE_RE.finditer(citation_text):
1146 # Expand comma lists AND dash ranges ("Sources 1-3") with the same
1147 # budget-checked helper as bracket citations, so "1-3" yields
1148 # {1, 2, 3} instead of just endpoints. "and"/"&" normalize to ",".
1149 nums = m.group("nums")
1150 nums_norm = re.sub(r"\s*(?:and|&)\s*", ",", nums, flags=re.IGNORECASE)
1151 _parse_bracket_content(nums_norm, indices, state)
1153 for m in url_matches:
1154 try:
1155 candidates = _url_citation_candidates(m, candidate_budget)
1156 except _URLCandidateBudgetExceeded:
1157 return indices, url_keys, True
1158 # Case-variant library routes (``/LIBRARY/Document/d2``): the regex
1159 # captures them (bare mentions and markdown destinations) but the
1160 # canonical key is prefix-case-sensitive, so also key the
1161 # prefix-normalized spelling. Applies to both branches — a
1162 # markdown destination is the stronger citation signal and must
1163 # not lose while the bare mention of the same route retains.
1164 for base in list(candidates):
1165 if _BARE_LIBRARY_PREFIX_RE.match(base):
1166 try:
1167 # /lib/ can expand to /library/; account before copying.
1168 candidate_budget.consume(len(base) + 4)
1169 except _URLCandidateBudgetExceeded:
1170 return indices, url_keys, True
1171 candidates.add(_normalize_bare_library_prefix(base))
1172 for candidate in candidates:
1173 if not candidate: 1173 ↛ 1174line 1173 didn't jump to line 1174 because the condition on line 1173 was never true
1174 continue
1175 try:
1176 key = canonical_url_key(candidate)
1177 except Exception:
1178 continue
1179 if key: 1179 ↛ 1172line 1179 didn't jump to line 1172 because the condition on line 1179 was always true
1180 url_keys.add(key)
1182 return indices, url_keys, state["indeterminate"]
1185def filter_cited_links(
1186 all_links: List[Dict],
1187 prose: Optional[str],
1188 mode: str = "fallback",
1189) -> List[Dict]:
1190 """Keep only links cited in ``prose`` (by index or URL).
1192 Modes: ``fallback`` (default) returns ``all_links`` when extraction
1193 is empty/indeterminate or nothing matches; ``strict`` returns only
1194 matches (possibly empty); ``disabled`` returns ``all_links``.
1195 Indeterminate input falls back to ``all_links`` in every mode.
1196 ``None`` resolves from the ``report.uncited_sources_mode`` setting.
1197 """
1198 mode = resolve_uncited_sources_mode(mode)
1199 if not all_links:
1200 return all_links
1201 if mode == "disabled" or not prose or not prose.strip():
1202 return all_links
1204 cited_indices, cited_urls, indeterminate = extract_cited_indices_and_urls(
1205 prose
1206 )
1207 if indeterminate:
1208 logger.warning(
1209 "filter_cited_links: indeterminate citation parse; "
1210 "falling back to all {} links",
1211 len(all_links),
1212 )
1213 return all_links
1214 if not cited_indices and not cited_urls:
1215 if mode == "fallback":
1216 logger.warning(
1217 "filter_cited_links: no citations parsed from prose "
1218 "({} links); falling back to full bibliography",
1219 len(all_links),
1220 )
1221 return all_links
1222 return []
1224 kept: List[Dict] = []
1225 for link in all_links:
1226 if not isinstance(link, dict): 1226 ↛ 1227line 1226 didn't jump to line 1227 because the condition on line 1226 was never true
1227 continue
1228 idx = link.get("index", "")
1229 try:
1230 idx_int = int(str(idx).strip())
1231 except (ValueError, TypeError):
1232 idx_int = None
1233 if idx_int is not None and idx_int in cited_indices:
1234 kept.append(link)
1235 continue
1236 raw = source_url_field(link)
1237 if isinstance(raw, str) and raw: 1237 ↛ 1225line 1237 didn't jump to line 1225 because the condition on line 1237 was always true
1238 try:
1239 key = canonical_url_key(raw)
1240 except Exception:
1241 key = ""
1242 if key and key in cited_urls:
1243 kept.append(link)
1245 if not kept and mode == "fallback":
1246 logger.warning(
1247 "filter_cited_links: {} cited indices / {} cited urls matched "
1248 "nothing in {} links; falling back to full bibliography",
1249 len(cited_indices),
1250 len(cited_urls),
1251 len(all_links),
1252 )
1253 return all_links
1254 return kept
1257def count_distinct_sources(all_links: List[Dict]) -> int:
1258 """How many distinct sources a report cites (document-level count).
1260 ``len(all_links_of_system)`` counts OCCURRENCES, not sources: the
1261 LangGraph collector stores one entry per ``(url, snippet)`` pair, and
1262 ``source_based`` / ``focused_iteration`` / ``topic_organization``
1263 extend the list with raw engine dicts and no URL dedup at all. Both
1264 shapes put one source in the list several times, so every "N sources"
1265 number must group at the document level rather than take a length.
1267 Counts by canonical URL key, so distinct library documents are counted
1268 once regardless of how many distinct chunks or views are cited.
1269 The relation across reporting layers is:
1270 distinct sources <= bibliography lines <= citation indices.
1271 """
1272 seen: set[str] = set()
1273 for link in all_links or []:
1274 if not isinstance(link, dict):
1275 continue
1276 raw = source_url_field(link)
1277 if not isinstance(raw, str):
1278 continue
1279 canon = canonical_url_key(raw)
1280 if canon:
1281 seen.add(canon)
1282 return len(seen)
1285def format_links_to_markdown(
1286 all_links: List[Dict],
1287 prose: Optional[str] = None,
1288 uncited_mode: Optional[str] = "fallback",
1289) -> str:
1290 parts: list[str] = []
1291 uncited_mode = resolve_uncited_sources_mode(uncited_mode)
1292 if prose and uncited_mode != "disabled":
1293 all_links = filter_cited_links(all_links, prose, mode=uncited_mode)
1294 logger.info(f"Formatting {len(all_links)} links to markdown...")
1296 if all_links:
1297 # Group and display ordinary URLs in canonical form. Timestamped
1298 # Wayback replays keep their complete spelling: their embedded
1299 # target path/query identifies the historical resource.
1300 # Group links by (canonical URL, chunk display URL).
1301 # Canonical key remains per-document so count_distinct_sources,
1302 # MCP sources, and news metrics do not inflate, while grouping on
1303 # chunk anchor ensures distinct cited chunks (/chunks#chunk-N) each
1304 # render their own Sources entry with their own anchor. Unanchored
1305 # views of the same document (e.g. /pdf and base route) merge together;
1306 # an unanchored /pdf view deliberately renders on its own line when
1307 # cited alongside chunks (reversing #5685's collapse so anchorless
1308 # indices honestly link to /pdf rather than mis-pointing at an arbitrary chunk).
1309 # Non-library sources display their canonical URL, including the
1310 # exact replay spelling preserved by canonical_url_key.
1311 url_to_indices: dict[tuple[str, str], list] = {}
1312 group_to_title: dict[tuple[str, str], str] = {}
1313 group_to_capture_date: dict[tuple[str, str], str] = {}
1314 # Keyed by canonical URL alone, unlike its neighbours. A document
1315 # cited through both its chunk anchor and its bare URL renders one
1316 # line per group, and its page title is the same on both; keying
1317 # this by the group left whichever line did not hold the key
1318 # showing the search listing's spelling.
1319 doc_to_fetched_title: dict[str, str] = {}
1320 group_to_quality: dict[tuple[str, str], int] = {}
1321 group_to_collection: dict[tuple[str, str], str] = {}
1322 group_to_display: dict[tuple[str, str], str] = {}
1323 for link in all_links:
1324 # Same skip as count_distinct_sources, so a malformed entry
1325 # costs one bibliography line rather than the whole block.
1326 if not isinstance(link, dict):
1327 continue
1328 raw = source_url_field(link)
1329 # Skipped, not coerced. These dicts reach here straight from
1330 # engine output on the non-LangGraph strategies, and
1331 # canonical_url_key raises on a non-str. Stringifying instead
1332 # renders a Python repr as a clickable URL and can merge two
1333 # distinct sources onto one citation — and it would disagree
1334 # with ``_citation_dedup_key``, which refuses a non-str link
1335 # outright and whose docstring requires the two to group
1336 # identically.
1337 if not isinstance(raw, str):
1338 continue
1339 canon = canonical_url_key(raw)
1340 if not canon:
1341 continue
1342 chunk_disp = preferred_chunk_display(raw) or _owned_chunk_display(
1343 link.get(CHUNK_DISPLAY_KEY), canon
1344 )
1345 if chunk_disp:
1346 key = (canon, chunk_disp)
1347 disp = chunk_disp
1348 else:
1349 key = (canon, "")
1350 disp = library_display_url(raw) or canon
1352 url_to_indices.setdefault(key, []).append(link.get("index", ""))
1353 group_to_title.setdefault(key, link.get("title", "Untitled"))
1354 if link.get("source_type") == "archive":
1355 capture_date = link.get("formatted_date")
1356 if isinstance(capture_date, str) and capture_date.strip():
1357 group_to_capture_date.setdefault(key, capture_date.strip())
1358 # A page's own title, recorded by the fetch ingest path beside a
1359 # citation whose stored title a search engine scraped. First
1360 # non-empty wins, as for quality and collection below.
1361 if canon not in doc_to_fetched_title:
1362 fetched = link.get(FETCHED_TITLE_KEY)
1363 if isinstance(fetched, str) and fetched.strip():
1364 doc_to_fetched_title[canon] = fetched.strip()
1365 # Prefer /pdf over bare /library/document/<id> for unanchored display
1366 curr_disp = group_to_display.get(key)
1367 if curr_disp is None or (
1368 curr_disp.endswith(canon) and "/pdf" in disp
1369 ):
1370 group_to_display[key] = disp
1372 # Track journal quality per group (first non-None wins)
1373 if key not in group_to_quality and link.get("journal_quality"): 1373 ↛ 1374line 1373 didn't jump to line 1374 because the condition on line 1373 was never true
1374 group_to_quality[key] = link["journal_quality"]
1375 # First non-empty collection name wins (mirrors title/quality).
1376 # Note: per-group collection tracking is display-only today and
1377 # anticipates #5722 encoding collection identity into chunk anchors.
1378 if key not in group_to_collection:
1379 metadata = link.get("metadata") or {}
1380 if not isinstance(metadata, dict):
1381 metadata = {}
1382 collection = metadata.get("collection_name")
1383 if collection:
1384 group_to_collection[key] = str(collection)
1386 # Emit each unique source once, in first-seen order.
1387 seen: set[tuple[str, str]] = set()
1388 for link in all_links:
1389 if not isinstance(link, dict):
1390 continue
1391 raw = source_url_field(link)
1392 if not isinstance(raw, str):
1393 continue
1394 canon = canonical_url_key(raw)
1395 if not canon:
1396 continue
1397 chunk_disp = preferred_chunk_display(raw) or _owned_chunk_display(
1398 link.get(CHUNK_DISPLAY_KEY), canon
1399 )
1400 key = (canon, chunk_disp) if chunk_disp else (canon, "")
1401 if key in seen:
1402 continue
1403 title = doc_to_fetched_title.get(canon) or group_to_title[key]
1404 # Coerced for the same reason the url is skipped: it comes
1405 # from the same engine dict, and ``.replace`` below raises on
1406 # a non-str, taking the whole Sources block with it.
1407 if title is not None and not isinstance(title, str):
1408 title = str(title)
1409 capture_date = group_to_capture_date.get(key)
1410 if capture_date:
1411 suffix = f" (archived {capture_date})"
1412 if not (title or "").endswith(suffix):
1413 title = f"{title or 'Untitled'}{suffix}"
1414 if title:
1415 title = _sanitize_sources_field(
1416 title.replace(LDR_APPENDED_SOURCES_SENTINEL, "")
1417 )
1418 # Indices arrive as int (from strategy enumeration) or str (from
1419 # _build_sources_markdown's fallback). Coerce so dedup collapses
1420 # 1 and "1", and sorted() doesn't TypeError on mixed types.
1421 indices = sorted(
1422 {str(i) for i in url_to_indices[key]},
1423 # ``isdigit()`` is True for non-ASCII digit characters that
1424 # ``int()`` rejects — e.g. the superscript "\u00b9" — so it
1425 # alone would raise ValueError here and crash the whole
1426 # bibliography. Require ASCII before converting.
1427 key=lambda s: (
1428 (0, int(s)) if s.isascii() and s.isdigit() else (1, s)
1429 ),
1430 )
1431 # Sanitised like the title, collection and URL. Indices are
1432 # LDR-internal enumerations today, but ``_create_documents``
1433 # PRESERVES a pre-existing ``index`` on a raw engine dict and
1434 # ``_build_sources_markdown`` reads it back out of persisted
1435 # ``original_data`` — so "no engine emits this key" is a
1436 # coincidence, not a control.
1437 #
1438 # ``quality_tag`` below is the one field still interpolated
1439 # raw. It reaches the renderer by the same persisted route,
1440 # and is safe only incidentally: the fall-through branch that
1441 # builds it uses ``repr()``, which escapes newlines and
1442 # non-printables. Incidental, not designed — do not remove
1443 # that ``repr()`` without sanitising here instead.
1444 indices = [_sanitize_sources_field(i) for i in indices]
1445 indices_str = f"[{', '.join(indices)}]"
1446 quality_tag = _format_quality_tag(group_to_quality.get(key))
1447 collection = group_to_collection.get(key, "")
1448 if collection:
1449 collection = _sanitize_sources_field(
1450 collection.replace(LDR_APPENDED_SOURCES_SENTINEL, "")
1451 )
1452 collection_line = (
1453 f" Collection: {collection}\n" if collection else ""
1454 )
1455 display = group_to_display[key]
1456 parts.append(
1457 f"{indices_str} {title}{quality_tag} "
1458 f"(source nr: {', '.join(map(str, indices))})\n"
1459 f" URL: {_sanitize_sources_field(display)}\n"
1460 f"{collection_line}"
1461 f"\n"
1462 )
1463 seen.add(key)
1465 parts.append("\n")
1467 return "".join(parts)
1470def format_findings(
1471 findings_list: List[Dict],
1472 synthesized_content: str,
1473 questions_by_iteration: Dict[int, List[str]],
1474 uncited_mode: Optional[str] = None,
1475) -> str:
1476 """Format findings into a detailed text output.
1478 Args:
1479 findings_list: List of finding dictionaries
1480 synthesized_content: The synthesized content from the LLM.
1481 questions_by_iteration: Dictionary mapping iteration numbers to lists of questions
1482 uncited_mode: ``report.uncited_sources_mode`` override. ``None``
1483 (default) resolves from settings. Only the global source
1484 dumps (top list and ``## ALL SOURCES`` footer) are filtered,
1485 using the synthesized content as the citation prose;
1486 per-section ``SOURCES USED IN THIS SECTION`` lists stay
1487 unfiltered.
1489 Returns:
1490 str: Formatted text output
1491 """
1492 logger.info(
1493 f"Inside format_findings utility. Findings count: {len(findings_list)}, Questions iterations: {len(questions_by_iteration)}"
1494 )
1495 parts: list[str] = []
1496 mode = resolve_uncited_sources_mode(uncited_mode)
1497 prose = (
1498 synthesized_content
1499 if isinstance(synthesized_content, str)
1500 else str(synthesized_content or "")
1501 )
1503 # Extract all sources from findings
1504 all_links = []
1505 for finding in findings_list:
1506 search_results = finding.get("search_results", [])
1507 if search_results:
1508 try:
1509 links = extract_links_from_search_results(search_results)
1510 all_links.extend(links)
1511 except Exception:
1512 logger.exception("Error processing search results/links")
1514 # Start with the synthesized content (passed as synthesized_content)
1515 parts.append(f"{synthesized_content}\n\n")
1517 # Add sources section after synthesized content if sources exist.
1518 # Filtered by the synthesized prose; per-section lists below stay
1519 # unfiltered.
1520 parts.append(
1521 format_links_to_markdown(all_links, prose=prose, uncited_mode=mode)
1522 )
1524 parts.append("\n\n") # Separator after synthesized content
1526 # Add Search Questions by Iteration section
1527 if questions_by_iteration:
1528 parts.append("## SEARCH QUESTIONS BY ITERATION\n")
1529 parts.append("\n")
1530 for iter_num, questions in questions_by_iteration.items():
1531 parts.append(f"\n #### Iteration {iter_num}:\n")
1532 for i, q in enumerate(questions, 1):
1533 parts.append(f"{i}. {q}\n")
1534 parts.append("\n\n\n")
1535 else:
1536 logger.warning("No questions by iteration found to format.")
1538 # Add Detailed Findings section
1539 if findings_list:
1540 parts.append("## DETAILED FINDINGS\n\n")
1541 logger.info(f"Formatting {len(findings_list)} detailed finding items.")
1543 for idx, finding in enumerate(findings_list):
1544 logger.debug(
1545 f"Formatting finding item {idx}. Keys: {list(finding.keys())}"
1546 )
1547 # Use .get() for safety
1548 phase = finding.get("phase", "Unknown Phase")
1549 content = finding.get("content", "No content available.")
1550 search_results = finding.get("search_results", [])
1552 # Phase header
1553 parts.append(f"\n### {phase}\n\n\n")
1555 question_displayed = False
1556 # If this is a follow-up phase, try to show the corresponding question
1557 if isinstance(phase, str) and phase.startswith("Follow-up"):
1558 try:
1559 phase_parts = phase.replace(
1560 "Follow-up Iteration ", ""
1561 ).split(".")
1562 if len(phase_parts) == 2:
1563 iteration = int(phase_parts[0])
1564 question_index = int(phase_parts[1]) - 1
1565 if (
1566 iteration in questions_by_iteration
1567 and 0
1568 <= question_index
1569 < len(questions_by_iteration[iteration])
1570 ):
1571 parts.append(
1572 f"#### {questions_by_iteration[iteration][question_index]}\n\n"
1573 )
1574 question_displayed = True
1575 else:
1576 logger.warning(
1577 f"Could not find matching question for phase: {phase}"
1578 )
1579 else:
1580 logger.warning(
1581 f"Could not parse iteration/index from phase: {phase}"
1582 )
1583 except ValueError:
1584 logger.warning(
1585 f"Could not parse iteration/index from phase: {phase}"
1586 )
1587 # Handle Sub-query phases from IterDRAG strategy
1588 elif isinstance(phase, str) and phase.startswith("Sub-query"):
1589 try:
1590 # Extract the index number from "Sub-query X"
1591 query_index = int(phase.replace("Sub-query ", "")) - 1
1592 # In IterDRAG, sub-queries are stored in iteration 0
1593 if 0 in questions_by_iteration and query_index < len(
1594 questions_by_iteration[0]
1595 ):
1596 parts.append(
1597 f"#### {questions_by_iteration[0][query_index]}\n\n"
1598 )
1599 question_displayed = True
1600 else:
1601 logger.warning(
1602 f"Could not find matching question for phase: {phase}"
1603 )
1604 except ValueError:
1605 logger.warning(
1606 f"Could not parse question index from phase: {phase}"
1607 )
1609 # If the question is in the finding itself, display it
1610 if (
1611 not question_displayed
1612 and "question" in finding
1613 and finding["question"]
1614 ):
1615 parts.append(f"### SEARCH QUESTION:\n{finding['question']}\n\n")
1617 # Content
1618 parts.append(f"\n\n{content}\n\n")
1620 # Search results if they exist
1621 if search_results:
1622 try:
1623 links = extract_links_from_search_results(search_results)
1624 if links:
1625 parts.append("### SOURCES USED IN THIS SECTION:\n")
1626 parts.append(format_links_to_markdown(links) + "\n\n")
1627 except Exception:
1628 logger.exception(
1629 f"Error processing search results/links for finding {idx}"
1630 )
1631 else:
1632 logger.debug(f"No search_results found for finding item {idx}.")
1634 parts.append(f"{'_' * 80}\n\n")
1635 else:
1636 logger.warning("No detailed findings found to format.")
1638 # Add summary of all sources at the end, filtered the same way.
1639 if all_links:
1640 parts.append("## ALL SOURCES:\n")
1641 parts.append(
1642 format_links_to_markdown(all_links, prose=prose, uncited_mode=mode)
1643 )
1644 else:
1645 logger.info("No unique sources found across all findings to list.")
1647 logger.info("Finished format_findings utility.")
1648 return "".join(parts)