Coverage for src/local_deep_research/text_optimization/citation_formatter.py: 97%
1104 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Citation formatter for adding hyperlinks and alternative citation styles."""
3import html
4import re
5import unicodedata
6import urllib.parse
7from array import array
8from enum import Enum
9from html.entities import html5
10from typing import Any, Dict, List, Optional, Tuple
11from urllib.parse import urlparse, urlsplit, urlunsplit
13from loguru import logger
14from slugify import slugify
16from ..content_fetcher.url_classifier import URLClassifier, URLType
18# Schemes a citation may carry into a markdown link. An allowlist rather than a
19# list of known-bad schemes: the formatter's job here is deciding what may be
20# rendered, and `javascript:`, `data:`, `vbscript:`, `blob:` and whatever comes
21# next all fail the same test without having to be enumerated. A relative URL
22# stays allowed, because a library document is cited as `/library/...`. A
23# destination that only looks scheme-less is still checked: its HTML-character-
24# reference-decoded form must pass too (`javascript:` renders as a live
25# `javascript:` href), and one that names a host (`//host/x`) is refused.
26_LINKABLE_URL_SCHEMES = frozenset({"http", "https"})
28# A scheme check alone is not enough. The destination is interpolated into
29# `](...)`, so anything markdown would read as the end of that destination
30# lets a source URL reopen markdown and write its own link past the gate.
31# Percent-encoding keeps the link resolvable, since a server decodes `%20`
32# back to a space. `[` and `]` are deliberately NOT encoded: they do not end a
33# destination, and encoding them would break IPv6 literals (`http://[::1]/x`).
34_DESTINATION_ESCAPES = {
35 " ": "%20",
36 "<": "%3C",
37 ">": "%3E",
38 '"': "%22",
39 "'": "%27",
40 "\\": "%5C",
41 "`": "%60",
42}
44# Parentheses are the exception, and they are why this is two tables rather
45# than one. CommonMark ends a bare destination at the first *unbalanced* `)`
46# and carries balanced pairs through unharmed, so encoding them unconditionally
47# rewrites ordinary URLs that nothing was wrong with -- Wikipedia alone titles
48# thousands of articles `..._(disambiguation)` -- and the reader is shown a URL
49# that is not the one the source served. A balanced destination is therefore
50# left verbatim, and only an unbalanced one is encoded, which is exactly the
51# case that would otherwise end early.
52_PARENTHESIS_ESCAPES = {"(": "%28", ")": "%29"}
55def _parentheses_are_balanced(url: str) -> bool:
56 """Whether markdown will carry every `(`/`)` in ``url`` inside the link."""
57 depth = 0
58 for character in url:
59 if character == "(":
60 depth += 1
61 elif character == ")":
62 depth -= 1
63 if depth < 0:
64 return False
65 return depth == 0
68def _destination_is_refused(form: str) -> bool:
69 """Whether one spelling of a destination fails the control or scheme rule."""
70 if any(
71 ord(character) < 0x20 or ord(character) == 0x7F for character in form
72 ):
73 return True
74 try:
75 scheme = urlsplit(form).scheme
76 except ValueError:
77 return True
78 if scheme:
79 return scheme not in _LINKABLE_URL_SCHEMES
80 # No scheme reads as relative, except where a browser takes the host from
81 # it: `//host/x`, and equally `///host/x` or `/\host/x`, which it resolves
82 # the same way and ``urlsplit`` does not. Rendered from a file on disk,
83 # an exported report resolves it to `file://host/x`.
84 return form.lstrip().replace("\\", "/").startswith("//")
87def _authority_head_has_userinfo(netloc: str, path: str) -> bool:
88 """Whether a linkable-scheme URL spells its authority without ``//``.
90 WHATWG-consuming renderers read ``https:user:pass@example.org/x`` — and
91 its ``https:/``, ``https:///`` and ``https:\\`` spellings — as the
92 authority ``user:pass@example.org``: any run of slashes or backslashes
93 stands in for the ``//``. ``urlsplit`` populates ``netloc`` only for the
94 ``//``-delimited form, so an @-in-netloc test alone lets those spellings
95 keep their userinfo. The authority-ish head — what follows the scheme,
96 past leading slashes or backslashes, up to the first ``/``, ``?`` or
97 ``#`` — is where those parsers read it; an @ there is treated as a
98 credential. An @ further along (``.../@author/...``, ``?q=a@b``) never
99 lands in it.
101 The head is rebuilt from the ``netloc`` and ``path`` that ``urlsplit``
102 parsed, never cut out of the input at the scheme's length. ``urlsplit``
103 drops leading spaces and C0 controls, and every tab and newline, before
104 it reads the scheme, as a browser does. `` https:/user:pass@host``
105 decodes to a leading space and then the URL, so a cut at
106 ``len("https") + 1`` started on the ``:``, found no ``@`` before the
107 next ``/``, and let the credential through. ``?`` and ``#`` need no
108 search here: ``urlsplit`` has already split them off.
109 """
110 rest = f"//{netloc}{path}" if netloc else path
111 head = rest.strip().lstrip("/\\").partition("/")[0]
112 return "@" in head
115def _spells_hidden_userinfo(candidate: str) -> bool:
116 """Whether a raw or entity-decoded reading of ``candidate`` hides userinfo.
118 Consulted after stripping literal userinfo, so every reading that finds
119 an @ is a spelling the plain parse missed:
120 an authority not delimited by ``//``, or one assembled by rendered HTML
121 entities — ``https://user:pass@host`` breaks the scheme token, parses
122 as scheme-less, and decodes into a live credential URL in a browser.
124 A reading that ``urlsplit`` rejects counts as hiding userinfo, just as
125 ``_destination_is_refused`` refuses one: the guard cannot vouch for what
126 it cannot parse. Stripping can produce such a reading.
127 ``https://[@v1.u:p]@v1.example/`` parses, but its userinfo is the
128 ``[`` before the first ``@``, and once that is removed what is left has
129 an unbalanced ``]``, which ``urlsplit`` rejects. A browser still decodes
130 that remainder into ``https://v1.u:p]@v1.example/``, whose userinfo is
131 ``v1.u:p]``.
132 """
133 try:
134 parsed = urlsplit(candidate)
135 except ValueError:
136 return True
137 if (
138 parsed.scheme in _LINKABLE_URL_SCHEMES
139 and "@" not in parsed.netloc
140 and _authority_head_has_userinfo(parsed.netloc, parsed.path)
141 ):
142 return True
143 decoded = html.unescape(candidate)
144 if decoded == candidate:
145 return False
146 try:
147 cooked = urlsplit(decoded)
148 except ValueError:
149 return True
150 return cooked.scheme in _LINKABLE_URL_SCHEMES and (
151 "@" in cooked.netloc
152 or _authority_head_has_userinfo(cooked.netloc, cooked.path)
153 )
156def _safe_link_destination(url: str) -> str:
157 """Return ``url`` ready to be interpolated into ``](...)``, or "" to refuse.
159 Four things have to hold, and what comes back is the string that was
160 checked rather than the one that came in:
162 1. No control character survives. ``urlsplit`` removes tabs and newlines
163 and strips leading C0 controls only in order to *parse*; they stay in
164 the string. A destination carrying a newline ends there and the rest is
165 emitted as document body. NUL is not stripped at all, so
166 ``java\x00script:`` parses as scheme-less and would otherwise pass.
167 2. The scheme is ``http``, ``https``, or absent. Looser than
168 ``CitationFormatter._is_linkable_url``, which requires a scheme and is
169 what SOURCE_TAGGED uses to decide whether to render a link at all; the
170 other modes keep linking the relative URLs a library document uses. A
171 scheme-less destination that names a host (``//host/x``) is refused.
172 3. HTTP userinfo is removed before the URL reaches a report or export.
173 It is a credential even when the destination itself is otherwise
174 safe, and it is refused rather than stripped when it is spelled
175 where the plain parser does not look — an authority not delimited
176 by ``//``, or one assembled from HTML entities — or when what is
177 left once it is removed no longer parses.
178 4. What markdown would read as syntax is percent-encoded, so the gate
179 cannot be walked around by ending the destination early.
181 Rules 1 and 2 are checked on the character-reference-decoded form too.
182 marked and Python-Markdown copy the destination into ``href`` as written,
183 and the HTML parser then decodes ``javascript:alert(1)`` into a live
184 ``javascript:`` URL. The decoded form is only checked, never emitted:
185 rewriting ``&`` would break every query string with more than one
186 parameter.
188 The rendered markdown reaches the browser through DOMPurify, which strips a
189 ``javascript:`` href before it reaches the DOM. That is one layer, and it is
190 the only one today: an exported .md / .qmd / .tex and every other markdown
191 consumer gets whatever this returns.
193 A refused destination leaves the citation unlinked rather than dropping
194 it: callers treat an empty URL as no URL, so the citation stays a plain
195 ``[N]`` bracket, or an unlinked tag such as ``[local-N]`` in
196 SOURCE_TAGGED mode.
197 """
198 candidate = url.strip()
199 if not candidate:
200 return ""
201 if _destination_is_refused(candidate) or _destination_is_refused(
202 html.unescape(candidate)
203 ):
204 return ""
205 parsed = urlsplit(candidate)
206 if parsed.scheme in _LINKABLE_URL_SCHEMES and "@" in parsed.netloc:
207 # A parseable authority keeps its host and loses its userinfo.
208 host = parsed.netloc.rpartition("@")[2]
209 if not host: 209 ↛ 210line 209 didn't jump to line 210 because the condition on line 209 was never true
210 return ""
211 candidate = urlunsplit(parsed._replace(netloc=host))
212 if _spells_hidden_userinfo(candidate):
213 # The userinfo is spelled where the plain parse does not look —
214 # without the ``//``, or through HTML entities that a rendered
215 # link decodes back into a live credential URL — or what is left
216 # no longer parses. Refuse rather than emit it.
217 return ""
218 escapes = dict(_DESTINATION_ESCAPES)
219 if not _parentheses_are_balanced(candidate):
220 escapes.update(_PARENTHESIS_ESCAPES)
221 return "".join(escapes.get(character, character) for character in candidate)
224# Marker emitted by ``IntegratedReportGenerator._format_final_report``
225# around the ``## Sources`` block it appends to the end of a detailed-mode
226# report. Looked up by ``find_sources_section`` so the splitter can locate
227# the appended sources section unambiguously, even when the LLM has
228# emitted its own ``## Sources`` (or ``References:`` etc.) header earlier
229# in the prose. The sentinel is an HTML comment, and renderMarkdown()
230# (services/ui.js) shows raw HTML in Markdown as literal text, so it must
231# not reach displayed report text: ``strip_appended_sources_sentinel``
232# removes it when a detailed report is saved, when a saved report is
233# assembled for display or export, and from news-feed findings.
234LDR_APPENDED_SOURCES_SENTINEL = "<!-- LDR_APPENDED_SOURCES_START -->"
236_SOURCES_SECTION_PATTERNS = [
237 re.compile(
238 r"^#{1,3}\s*(?:Sources|References|Bibliography|Citations)",
239 re.MULTILINE | re.IGNORECASE,
240 ),
241 re.compile(
242 r"^(?:Sources|References|Bibliography|Citations):?\s*$",
243 re.MULTILINE | re.IGNORECASE,
244 ),
245]
248def find_sources_section(
249 content: str, *, trust_sentinel: bool = True
250) -> Tuple[int, bool]:
251 """Locate the sources/references section in *content*.
253 Returns ``(position, on_sentinel)``. ``position`` is the character
254 offset of the section start, or ``-1`` if no section is found.
255 ``on_sentinel`` is ``True`` iff the unique sentinel emitted by the
256 report generator was matched (in which case the boundary is correct
257 by construction); ``False`` if the position came from the legacy
258 regex patterns (which can match an early ``## Sources`` line inside
259 the answer body and need an over-strip safety check downstream).
261 The sentinel is matched at its **last** occurrence (``rfind``) so a
262 spoofed earlier occurrence inside LLM-generated prose — e.g. the
263 LLM quoting the marker string verbatim while summarizing the
264 codebase — cannot silently truncate the answer body or disable the
265 over-strip safety check. Only a sentinel on its own line is trusted;
266 inline occurrences are skipped while searching backwards. The report
267 generator only ever emits the sentinel once at the end of the assembled
268 report, so the last standalone occurrence is the trusted one.
270 When ``trust_sentinel`` is ``False`` (e.g. raw LLM output from the
271 quick-mode save path, which never adds the sentinel itself), the
272 sentinel branch is skipped — only the legacy regex patterns run, so
273 a spurious ``on_sentinel=True`` cannot arise from the LLM quoting
274 the marker.
275 """
276 if trust_sentinel:
277 sentinel_pos = content.rfind(LDR_APPENDED_SOURCES_SENTINEL)
278 while sentinel_pos != -1:
279 preceded = sentinel_pos == 0 or content[sentinel_pos - 1] in (
280 "\n",
281 "\r",
282 )
283 after = content[sentinel_pos + len(LDR_APPENDED_SOURCES_SENTINEL) :]
284 followed = not after or after[0] in ("\n", "\r")
285 if preceded and followed:
286 return sentinel_pos, True
287 sentinel_pos = content.rfind(
288 LDR_APPENDED_SOURCES_SENTINEL, 0, sentinel_pos
289 )
290 for pattern in _SOURCES_SECTION_PATTERNS:
291 match = pattern.search(content)
292 if match:
293 return match.start(), False
294 return -1, False
297def strip_appended_sources_sentinel(content: str) -> str:
298 """Remove the report generator's appended-sources sentinel from *content*.
300 Only the occurrence ``find_sources_section`` trusts is removed: the last
301 one that stands on its own line, which is the one
302 ``IntegratedReportGenerator`` emitted. The line breaks around it are kept.
303 A copy quoted inside a line, or an earlier one, is report text and is
304 left as written.
306 The sentinel is an HTML comment. renderMarkdown() (services/ui.js) shows
307 raw HTML in Markdown as literal text, so a sentinel left in report text
308 would appear on the page. The detailed-mode save site keeps the whole
309 report, Sources block and sentinel included, in
310 ``CitationMode.NO_HYPERLINKS`` mode and when the citation formatter
311 raises; it now removes the sentinel before saving, and reports saved
312 that way earlier still contain it.
314 Anything that is not a non-empty string is returned unchanged.
315 """
316 if not isinstance(content, str) or not content:
317 return content
318 sentinel_start, on_sentinel = find_sources_section(content)
319 if not on_sentinel:
320 return content
321 return (
322 content[:sentinel_start]
323 + content[sentinel_start + len(LDR_APPENDED_SOURCES_SENTINEL) :]
324 )
327_BIB_SOURCES_PATTERN = re.compile(
328 r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
329 re.MULTILINE,
330)
333# Single-pass character map. A sequential replace() chain would re-escape
334# the braces its own replacements introduce — the same class of bug that
335# _escape_latex_text's single regex pass avoids below, where "\\" becomes
336# "\textbackslash{}" without a later brace rule mangling it again.
337_BIBTEX_ESCAPES = {
338 "\\": r"\textbackslash{}",
339 # Balanced pairs, not backslash escapes: BibTeX's field scanner counts
340 # brace CHARACTERS and has no escape mechanism, so "\{" is still an open
341 # brace to it. A backslashed brace left the value unbalanced, and the
342 # scanner ran to EOF swallowing the rest of the .bib.
343 "{": r"{\textbraceleft}",
344 "}": r"{\textbraceright}",
345 "&": r"\&",
346 "%": r"\%",
347 "$": r"\$",
348 "#": r"\#",
349 "_": r"\_",
350 "~": r"\textasciitilde{}",
351 "^": r"\textasciicircum{}",
352 '"': "'",
353}
356# U+0085 NEL already falls inside the 0x7F-0x9F range below; only these two
357# separators sit outside it.
358_UNICODE_LINE_SEPARATORS = "\u2028\u2029"
361def is_line_breaking_char(char: str) -> bool:
362 """True if *char* can begin a new line for some consumer of our output.
364 ONE definition, imported by every producer. The recurring bug in this
365 area has been fixing one instance and leaving a sibling — CR handled
366 but not NEL, the URL sanitised but not the citation index — so the
367 union is deliberately wider than any single consumer needs: C0 and DEL
368 for everything, C1 because such a codepoint is never legitimate text,
369 and U+2028/U+2029 because CSS Text makes them mandatory breaks in
370 rendered HTML even though Python's ``re`` and ``str.splitlines()``
371 ignore them.
373 Imported by ``search_utilities._sanitize_sources_field``;
374 ``test_bibliography_escaping`` asserts the two agree on every BMP
375 codepoint, so the claim cannot rot into a comment again.
376 """
377 code = ord(char)
378 return (
379 code < 0x20 or 0x7F <= code <= 0x9F or char in _UNICODE_LINE_SEPARATORS
380 )
383def _escape_bibtex(value: str) -> str:
384 r"""Escape a value for interpolation into a BibTeX field.
386 Titles come from search-result metadata — whatever a page calls itself
387 — and were previously written into ``title = "{...}"`` raw. A title
388 ending the field and opening another (``Benign}", url = "...", note =
389 "{x``) injected an attacker-controlled ``url`` that BibTeX preferred,
390 and an unbalanced brace ran the parser on into the rest of the file.
391 """
392 return "".join(
393 # Control characters are flattened, not escaped. The .bib is
394 # embedded in a ```bibtex markdown fence by the Quarto exporter,
395 # and a bare CR — which ``_BIB_SOURCES_PATTERN``'s ``.`` matches,
396 # since only ``\n`` is excluded — closes that fence once a reader
397 # normalises CRLF, turning the remainder into live markup.
398 # ``_safe_bibtex_url`` already rejects these; this matches it.
399 " " if is_line_breaking_char(ch) else _BIBTEX_ESCAPES.get(ch, ch)
400 for ch in value
401 )
404_LATEX_TEXT_SPECIALS: dict[str, str] = {
405 "\\": r"\textbackslash{}",
406 "&": r"\&",
407 "%": r"\%",
408 "$": r"\$",
409 "#": r"\#",
410 "_": r"\_",
411 "{": r"\{",
412 "}": r"\}",
413 "~": r"\textasciitilde{}",
414 "^": r"\textasciicircum{}",
415}
416_LATEX_TEXT_SPECIALS_RE = re.compile(
417 "[" + re.escape("".join(_LATEX_TEXT_SPECIALS)) + "]"
418)
421def _escape_latex_text(text: str) -> str:
422 r"""Escape every LaTeX-special character in one pass.
424 One regex pass is load-bearing. Sequential ``str.replace`` passes
425 re-process their own output: the braces in ``\textbackslash{}`` get
426 escaped again and a literal backslash comes out as the invalid
427 ``\textbackslash\{\}``. ``re.sub`` never rescans replacement text,
428 so each special character is replaced exactly once.
429 """
430 return _LATEX_TEXT_SPECIALS_RE.sub(
431 lambda m: _LATEX_TEXT_SPECIALS[m.group()], text
432 )
435# Matches the leading '#' markers of an ATX heading so the heading TEXT
436# can be escaped while the marker itself survives for the
437# heading-conversion pass in export_to_latex.
438_HEADING_PREFIX_RE = re.compile(r"^(\s*#+\s+)(.*)$")
441def _safe_bibtex_url(value: str) -> str:
442 r"""Return *value* if it is safe to place in ``url = {...}``, else "".
444 Escaping is wrong for a URL — a ``\%`` would corrupt a percent-encoded
445 octet. Nor may the offending characters simply be DROPPED: removing a
446 backslash from ``//trusted.example\@evil.example/p`` turns the trusted
447 host into userinfo, so the exported link points at a different host
448 than the one the reader vetted in the report. A URL containing a
449 breakout character is not a URL to repair — it is omitted, and the
450 caller falls through to its existing URL-less branch.
451 """
452 if any(ch in '{}"\\ ' or is_line_breaking_char(ch) for ch in value):
453 # Local-document sources legitimately trip this: a LangChain
454 # retriever sets a result's url from its ``source`` metadata, which
455 # is often a filesystem path carrying spaces or backslashes.
456 # Dropping the link silently would leave the user guessing why a
457 # bibliography entry has none, so record it.
458 # Truncated: a hostile result can carry a megabyte-long URL, and
459 # this fires once per affected entry — and the Quarto exporter
460 # builds the bibliography twice per export, so it fires twice per
461 # entry in practice. ``!r`` keeps control characters escaped.
462 logger.debug(
463 "Omitting unsafe bibliography URL: {!r}{}",
464 value[:200],
465 "…" if len(value) > 200 else "",
466 )
467 return ""
468 return value
471def _iter_citation_group(group: str):
472 r"""Yield each index on one Sources line (``"1, 2, 3"`` -> "1", "2", "3").
474 ``format_links_to_markdown`` collapses every citation of one source
475 onto a single line carrying all of its indices, so a single-index
476 pattern drops such a source from the bibliography entirely. Exporters
477 emit one entry per index so each ``[N]`` in the body resolves.
479 Indices are yielded as STRINGS, exactly as written: the body citations
480 are rewritten from the same text, so converting ``007`` to ``7`` here
481 would key the entry ``ref7`` while the body still cites ``ref007``.
482 No digit filtering either — the pattern's ``\d`` matches only decimal
483 digits (category ``Nd``), which ``int()`` always accepts, so filtering
484 could only discard valid indices such as Arabic-Indic ones.
485 """
486 for part in group.split(","):
487 part = part.strip()
488 if part: 488 ↛ 486line 488 didn't jump to line 486 because the condition on line 488 was always true
489 yield part
492def _sort_key(index: str):
493 """Order bibliography entries numerically where possible."""
494 try:
495 return (0, int(index), "")
496 except ValueError:
497 return (1, 0, index)
500def _collect_bibliography_entries(content: str) -> dict:
501 """Map each citation index to the ``(title, url)`` it names.
503 Selection rule: a URL-bearing entry never loses to a URL-less one, and
504 among URL-bearing entries the LATER occurrence wins.
506 Both halves matter. A report repeats its Sources block once per section
507 and then again as a combined block, and the combined block is both last
508 and authoritative — so later-wins. And the group-aware pattern can match
509 a line of prose that opens with ``[1, 2]`` where a single-index pattern
510 could not; such a line carries no ``URL:``, whereas every line
511 ``format_links_to_markdown`` renders does, so the URL preference is what
512 keeps prose from taking a citation key.
514 Deliberately NOT scoped to a Sources section. Bounding the block at
515 markers like ``\n---`` looked safer but silently emptied the
516 bibliography whenever a thematic break, a setext underline or a table
517 delimiter row came first — while the body was still rewritten to cite
518 entries that no longer existed.
519 """
520 entries: dict[str, tuple[str, str]] = {}
521 for match in _BIB_SOURCES_PATTERN.finditer(content):
522 title = match.group(2).strip()
523 url = match.group(3).strip() if match.group(3) else ""
524 for index in _iter_citation_group(match.group(1)):
525 # Prefer an entry carrying a URL: a real Sources line has one,
526 # a prose false positive does not.
527 if url or index not in entries: 527 ↛ 524line 527 didn't jump to line 524 because the condition on line 527 was always true
528 entries[index] = (title, url)
529 return entries
532class CitationMode(Enum):
533 """Available citation formatting modes."""
535 NUMBER_HYPERLINKS = "number_hyperlinks" # [1] with hyperlinks
536 DOMAIN_HYPERLINKS = "domain_hyperlinks" # [arxiv.org] with hyperlinks
537 DOMAIN_ID_HYPERLINKS = (
538 "domain_id_hyperlinks" # [arxiv.org] or [arxiv.org-1] with smart IDs
539 )
540 DOMAIN_ID_ALWAYS_HYPERLINKS = (
541 "domain_id_always_hyperlinks" # [arxiv.org-1] always with IDs
542 )
543 SOURCE_TAGGED_HYPERLINKS = "source_tagged_hyperlinks"
544 """Preserve the global citation number and prefix it with a short source
545 tag derived from the URL: known academic sources via ``URLClassifier``
546 (``arxiv-7``, ``pubmed-3``), domain otherwise (``nytimes.com-9``), and
547 ``local-N`` for empty / local URLs. Unlike DOMAIN_ID_* modes the
548 suffix is the original citation number, so labels never collide and
549 match the bibliography order: ``[1]`` arxiv + ``[2]`` openai + ``[3]``
550 arxiv -> ``arxiv-1``, ``openai-2``, ``arxiv-3``."""
551 NO_HYPERLINKS = "no_hyperlinks" # [1] without hyperlinks
554class CitationFormatter:
555 """Formats citations in markdown documents with various styles."""
557 def __init__(self, mode: CitationMode = CitationMode.NUMBER_HYPERLINKS):
558 self.mode = mode
559 # Use negative lookbehind and lookahead to avoid matching already formatted citations
560 # Also match Unicode lenticular brackets 【】 (U+3010 and U+3011) that LLMs sometimes generate
561 self.citation_pattern = re.compile(
562 r"(?<![\[【])[\[【](\d+)[\]】](?![\]】])"
563 )
564 self.comma_citation_pattern = re.compile(
565 r"[\[【](\d+(?:\s*,\s*\d+)+)[\]】]" # (#6944) space allowed BEFORE the comma too — a hand-edited `[[2 ,3]]` must stay a group
566 )
567 # Also match "Source X" or "source X" patterns
568 self.source_word_pattern = re.compile(r"\b[Ss]ource\s+(\d+)\b")
569 self.sources_pattern = re.compile(
570 r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
571 re.MULTILINE,
572 )
574 def _create_source_word_replacer(self, formatter_func):
575 """Create a replacement function for 'Source X' patterns.
577 Args:
578 formatter_func: A function that takes citation_num and returns formatted text
580 Returns:
581 A replacement function for use with regex sub
582 """
584 def replace_source_word(match):
585 citation_num = match.group(1)
586 return formatter_func(citation_num)
588 return replace_source_word
590 def _create_citation_formatter(self, sources_dict, format_pattern):
591 """Create a formatter function for citations.
593 Args:
594 sources_dict: Dictionary mapping citation numbers to data
595 format_pattern: A callable that takes (citation_num, data) and returns formatted string
597 Returns:
598 A function that formats citations or returns fallback
599 """
601 def formatter(citation_num):
602 if citation_num in sources_dict:
603 data = sources_dict[citation_num]
604 return format_pattern(citation_num, data)
605 return f"[{citation_num}]"
607 return formatter
609 def _replace_comma_citations(self, content, lookup, format_one):
610 """Replace comma-separated citations like [1, 2, 3] using *lookup* and *format_one*.
612 Args:
613 content: Text to process
614 lookup: Dict mapping citation number (str) to data
615 format_one: ``(num, data) -> str`` callback that formats a single citation
616 """
618 def _replacer(match):
619 nums = [n.strip() for n in match.group(1).split(",")]
620 parts = []
621 for num in nums:
622 if num in lookup:
623 parts.append(format_one(num, lookup[num]))
624 else:
625 parts.append(f"[{num}]")
626 return "".join(parts)
628 return self.comma_citation_pattern.sub(_replacer, content)
630 def format_document(self, content: str) -> str:
631 """Format citations in *content* and return the full document.
633 This is the legacy single-string entry point: it concatenates
634 the answer half and the sources half produced by
635 :meth:`format_document_split`. New code that needs to know
636 where the boundary between answer and sources actually fell
637 should use :meth:`format_document_split` instead so the boundary
638 is returned explicitly (no re-parsing of the concatenated output).
639 """
640 formatted_answer, sources_md, _on_sentinel = self.format_document_split(
641 content
642 )
643 return formatted_answer + sources_md
645 def format_document_split(
646 self, content: str, *, trust_sentinel: bool = True
647 ) -> Tuple[str, str, bool]:
648 """Format citations and return ``(answer, sources_md, on_sentinel)``.
650 The boundary between the LLM's answer and the trailing Sources
651 section is computed inside this method. Callers that only want
652 the answer (e.g. the chat-mode save site) get a clean split
653 without re-applying a regex on concatenated output downstream.
654 ``on_sentinel`` mirrors the second element of
655 :func:`find_sources_section`'s return — ``True`` iff the split
656 boundary came from the appended-sources sentinel emitted by
657 ``IntegratedReportGenerator``, in which case the boundary is
658 correct by construction and downstream over-strip safety checks
659 should be skipped. Pass ``trust_sentinel=False`` for raw LLM
660 output (quick-mode save path) where the sentinel would only
661 appear if the LLM quoted the marker itself.
663 Returns ``(content, "", False)`` when the formatter is in
664 NO_HYPERLINKS mode or when no Sources section can be found in
665 ``content``.
666 """
667 if self.mode == CitationMode.NO_HYPERLINKS:
668 return content, "", False
670 sources_start, on_sentinel = find_sources_section(
671 content, trust_sentinel=trust_sentinel
672 )
673 if sources_start == -1:
674 return content, "", False
676 document_content = content[:sources_start]
677 sources_content = content[sources_start:]
679 sources = self._parse_sources(sources_content)
681 if self.mode == CitationMode.NUMBER_HYPERLINKS:
682 formatted_content = self._format_number_hyperlinks(
683 document_content, sources
684 )
685 elif self.mode == CitationMode.DOMAIN_HYPERLINKS:
686 formatted_content = self._format_domain_hyperlinks(
687 document_content, sources
688 )
689 elif self.mode == CitationMode.DOMAIN_ID_HYPERLINKS:
690 formatted_content = self._format_domain_id_hyperlinks(
691 document_content, sources
692 )
693 elif self.mode == CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS:
694 formatted_content = self._format_domain_id_always_hyperlinks(
695 document_content, sources
696 )
697 elif self.mode == CitationMode.SOURCE_TAGGED_HYPERLINKS: 697 ↛ 704line 697 didn't jump to line 704 because the condition on line 697 was always true
698 formatted_content = self._format_source_tagged_hyperlinks(
699 document_content,
700 sources,
701 self._parse_collections(sources_content),
702 )
703 else:
704 formatted_content = document_content
706 return formatted_content, sources_content, on_sentinel
708 def apply_inline_hyperlinks(
709 self, content: str, sources: List[Dict[str, Any]]
710 ) -> str:
711 """Hyperlink ``[N]`` refs using a structured source list.
713 Dispatches on ``self.mode`` so the user's chosen citation
714 format (Settings → Report → Citation Format) is honored on
715 the fallback path the same way it is in
716 :meth:`format_document_split`. Inherits all the existing
717 per-mode guards (lookbehind/lookahead against ``[[1]]``,
718 comma-list handling like ``[1,2,3]``, ``Source N`` word form,
719 missing-index pass-through, lenticular bracket support).
721 Used as the safe fallback at save time when the LLM does NOT
722 emit a Sources section in its prose — the structured source
723 list (e.g. ``search_system.all_links_of_system``) is the
724 canonical source of URLs and indices.
725 """
726 if not content or not sources:
727 return content or ""
728 if self.mode == CitationMode.NO_HYPERLINKS:
729 return content
731 # Search-engine result dicts use either "url" or "link" for the
732 # destination — Searxng emits {"link": ..., "title": ..., "snippet": ...}
733 # (search_engine_searxng.py:538) and other engines use "url".
734 # Looking up only `s["url"]` silently dropped every Searxng-sourced
735 # citation, leaving the answer body with plain `[N]` brackets even
736 # though the Sources section beneath was fully populated. Accept
737 # both keys so the hyperlink fallback works regardless of engine.
738 def _src_url(s):
739 return _safe_link_destination(s.get("url") or s.get("link") or "")
741 adapted: Dict[str, Tuple[str, str]] = {
742 str(s["index"]): (s.get("title", "Untitled"), _src_url(s))
743 for s in sources
744 if _src_url(s) and s.get("index") is not None
745 }
746 if not adapted:
747 return content
749 # Per-mode dispatch — mirrors format_document_split so the user's
750 # chosen citation format applies on this fallback path too.
751 # Previously this was hard-coded to _format_number_hyperlinks,
752 # which meant chat-mode answers (which always hit this fallback
753 # because the langgraph-agent synthesis doesn't emit a ## Sources
754 # block in its prose) ignored the report.citation_format setting
755 # entirely — every chat answer came out as [[N]](url) even when
756 # the user picked domain-based or source-tagged formatting.
757 if self.mode == CitationMode.DOMAIN_HYPERLINKS:
758 return self._format_domain_hyperlinks(content, adapted)
759 if self.mode == CitationMode.DOMAIN_ID_HYPERLINKS:
760 return self._format_domain_id_hyperlinks(content, adapted)
761 if self.mode == CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS:
762 return self._format_domain_id_always_hyperlinks(content, adapted)
763 if self.mode == CitationMode.SOURCE_TAGGED_HYPERLINKS:
764 # Pull collection names off the structured source dicts
765 # (format_links_to_markdown uses the same shape:
766 # link["metadata"]["collection_name"]) so the SOURCE_TAGGED
767 # formatter can surface library/RAG tags as the citation
768 # label when present.
769 collections: Dict[str, str] = {}
770 for s in sources:
771 idx = s.get("index")
772 if idx is None: 772 ↛ 773line 772 didn't jump to line 773 because the condition on line 772 was never true
773 continue
774 meta = s.get("metadata") or {}
775 coll = meta.get("collection_name")
776 if coll:
777 collections.setdefault(str(idx), str(coll))
778 return self._format_source_tagged_hyperlinks(
779 content, adapted, collections
780 )
781 # NUMBER_HYPERLINKS is the default and the catch-all for any
782 # mode added later that doesn't have an explicit branch above.
783 return self._format_number_hyperlinks(content, adapted)
785 def _find_sources_section(
786 self, content: str, trust_sentinel: bool = True
787 ) -> Tuple[int, bool]:
788 """Find the start of the sources/references section.
790 Returns ``(position, on_sentinel)`` — see
791 :func:`find_sources_section` for the semantics. ``on_sentinel``
792 is True iff the boundary came from the unique sentinel emitted
793 by the report generator (i.e. it is correct by construction).
794 """
795 return find_sources_section(content, trust_sentinel=trust_sentinel)
797 def _parse_sources(
798 self, sources_content: str
799 ) -> Dict[str, Tuple[str, str]]:
800 """
801 Parse sources section to extract citation numbers, titles, and URLs.
803 Returns:
804 Dictionary mapping citation number to (title, url) tuple
805 """
806 sources = {}
807 matches = list(self.sources_pattern.finditer(sources_content))
809 for match in matches:
810 citation_nums_str = match.group(1)
811 title = match.group(2).strip()
812 url = _safe_link_destination(match.group(3) or "")
814 # Handle comma-separated citation numbers like [36, 3]
815 # Split by comma and strip whitespace
816 individual_nums = [
817 num.strip() for num in citation_nums_str.split(",")
818 ]
820 # Add an entry for each individual number
821 for num in individual_nums:
822 sources[num] = (title, url)
824 return sources
826 def _format_number_hyperlinks(
827 self, content: str, sources: Dict[str, Tuple[str, str]]
828 ) -> str:
829 """Replace [1] with hyperlinked version where only the number is linked."""
830 # Filter sources that have URLs
831 url_sources = {
832 num: (title, url) for num, (title, url) in sources.items() if url
833 }
835 # Create formatter for citations with number hyperlinks
836 def format_number_link(citation_num, data):
837 _, url = data
838 return f"[[{citation_num}]]({url})"
840 # Handle comma-separated citations like [1, 2, 3]
841 content = self._replace_comma_citations(
842 content, url_sources, format_number_link
843 )
845 formatter = self._create_citation_formatter(
846 url_sources, format_number_link
847 )
849 # Handle individual citations
850 def replace_citation(match):
851 return (
852 formatter(match.group(1))
853 if match.group(1) in url_sources
854 else match.group(0)
855 )
857 content = self.citation_pattern.sub(replace_citation, content)
859 # Also handle "Source X" patterns
860 return self.source_word_pattern.sub(
861 self._create_source_word_replacer(formatter), content
862 )
864 def _format_domain_hyperlinks(
865 self, content: str, sources: Dict[str, Tuple[str, str]]
866 ) -> str:
867 """Replace [1] with [domain.com] hyperlinked version."""
869 # Filter sources that have URLs
870 url_sources = {
871 num: (title, url) for num, (title, url) in sources.items() if url
872 }
874 # Pre-compute labels so [1, 2, 3] lists and standalone [N] refs
875 # use the same string for a given citation. Previously this
876 # closure called _extract_domain on every match, so a relative
877 # URL (``/library/document/...``) produced ``[[]]`` for every
878 # occurrence. _citation_label falls back to a slugified title
879 # for relative URLs, keeping the label non-empty and readable.
880 label_cache: Dict[str, str] = {
881 num: self._citation_label(num, title, url)
882 for num, (title, url) in url_sources.items()
883 }
885 # Create formatter for citations with domain hyperlinks
886 def format_domain_link(citation_num, data):
887 _, url = data
888 label = label_cache[citation_num]
889 return f"[[{label}]]({url})"
891 # Handle comma-separated citations like [1, 2, 3]
892 content = self._replace_comma_citations(
893 content, url_sources, format_domain_link
894 )
896 formatter = self._create_citation_formatter(
897 url_sources, format_domain_link
898 )
900 # Handle individual citations
901 def replace_citation(match):
902 return (
903 formatter(match.group(1))
904 if match.group(1) in url_sources
905 else match.group(0)
906 )
908 content = self.citation_pattern.sub(replace_citation, content)
910 # Also handle "Source X" patterns
911 return self.source_word_pattern.sub(
912 self._create_source_word_replacer(formatter), content
913 )
915 def _format_domain_id_hyperlinks(
916 self, content: str, sources: Dict[str, Tuple[str, str]]
917 ) -> str:
918 """Replace [1] with [domain.com-1] hyperlinked version with hyphen-separated IDs."""
919 # First, create a mapping of domains to their citation numbers.
920 # _citation_label (vs raw _extract_domain) means relative URLs
921 # like ``/library/document/...`` get a slugified title as the
922 # "domain" instead of an empty string. Without this, a RAG
923 # result with one citation rendered as ``[[]](url)`` (the
924 # label disappeared), and a multi-citation set rendered as
925 # ``[[-1]](url)`` (the orphaned ``-N`` suffix leaked through).
926 domain_citations: dict[str, list[Any]] = {}
928 for citation_num, (title, url) in sources.items():
929 if url:
930 domain = self._citation_label(citation_num, title, url)
931 if domain not in domain_citations:
932 domain_citations[domain] = []
933 domain_citations[domain].append((citation_num, url))
935 # Create a mapping from citation number to domain with ID
936 citation_to_domain_id = {}
937 for domain, citations in domain_citations.items():
938 if len(citations) > 1:
939 # Multiple citations from same domain - add hyphen and number
940 for idx, (citation_num, url) in enumerate(citations, 1):
941 citation_to_domain_id[citation_num] = (
942 f"{domain}-{idx}",
943 url,
944 )
945 else:
946 # Single citation from domain - no ID needed
947 citation_num, url = citations[0]
948 citation_to_domain_id[citation_num] = (domain, url)
950 # Create formatter for citations with domain_id hyperlinks
951 def format_domain_id_link(citation_num, data):
952 domain_id, url = data
953 return f"[[{domain_id}]]({url})"
955 # Handle comma-separated citations
956 content = self._replace_comma_citations(
957 content, citation_to_domain_id, format_domain_id_link
958 )
960 formatter = self._create_citation_formatter(
961 citation_to_domain_id, format_domain_id_link
962 )
964 # Handle individual citations
965 def replace_citation(match):
966 return (
967 formatter(match.group(1))
968 if match.group(1) in citation_to_domain_id
969 else match.group(0)
970 )
972 content = self.citation_pattern.sub(replace_citation, content)
974 # Also handle "Source X" patterns
975 return self.source_word_pattern.sub(
976 self._create_source_word_replacer(formatter), content
977 )
979 def _format_domain_id_always_hyperlinks(
980 self, content: str, sources: Dict[str, Tuple[str, str]]
981 ) -> str:
982 """Replace [1] with [domain.com-1] hyperlinked version, always with IDs."""
983 # Use _citation_label so relative URLs (e.g. RAG /library/document/...
984 # results) produce a slugified-title prefix instead of an empty
985 # string. Without this, every label here collapses to ``-N``
986 # (e.g. ``[[-1]](url)``) and the user has no idea what the link
987 # references. See _format_domain_id_hyperlinks for the same fix.
988 domain_citations: dict[str, list[Any]] = {}
990 for citation_num, (title, url) in sources.items():
991 if url:
992 domain = self._citation_label(citation_num, title, url)
993 if domain not in domain_citations:
994 domain_citations[domain] = []
995 domain_citations[domain].append((citation_num, url))
997 # Create a mapping from citation number to domain with ID
998 citation_to_domain_id = {}
999 for domain, citations in domain_citations.items():
1000 # Always add hyphen and number for consistency
1001 for idx, (citation_num, url) in enumerate(citations, 1):
1002 citation_to_domain_id[citation_num] = (f"{domain}-{idx}", url)
1004 # Create formatter for citations with domain_id hyperlinks
1005 def format_domain_id_link(citation_num, data):
1006 domain_id, url = data
1007 return f"[[{domain_id}]]({url})"
1009 # Handle comma-separated citations
1010 content = self._replace_comma_citations(
1011 content, citation_to_domain_id, format_domain_id_link
1012 )
1014 formatter = self._create_citation_formatter(
1015 citation_to_domain_id, format_domain_id_link
1016 )
1018 # Handle individual citations
1019 def replace_citation(match):
1020 return (
1021 formatter(match.group(1))
1022 if match.group(1) in citation_to_domain_id
1023 else match.group(0)
1024 )
1026 content = self.citation_pattern.sub(replace_citation, content)
1028 # Also handle "Source X" patterns
1029 return self.source_word_pattern.sub(
1030 self._create_source_word_replacer(formatter), content
1031 )
1033 # Sources section may carry a "Collection: <name>" line for RAG /
1034 # library hits (emitted by ``utilities/search_utilities.format_links_to_markdown``).
1035 # Read one line at a time: overlapping multiline whitespace patterns
1036 # can repeatedly rescan an entire run of blank lines.
1037 _collection_header_pattern = re.compile(r"^\[(\d+(?:,\s*\d+)*)\]")
1039 def _parse_collections(self, sources_content: str) -> Dict[str, str]:
1040 """Extract ``{citation_num: collection_name}`` from a sources
1041 block. Returns an empty dict when no ``Collection:`` lines exist
1042 — the absence of collection info is the common case (web URLs)
1043 and must never raise."""
1044 collections: Dict[str, str] = {}
1045 citation_nums = ()
1046 for line in sources_content.split("\n"):
1047 if line.startswith("["):
1048 match = self._collection_header_pattern.match(line)
1049 citation_nums = (
1050 tuple(_iter_citation_group(match.group(1))) if match else ()
1051 )
1052 continue
1053 field = line.strip()
1054 if citation_nums and field.startswith("Collection:"):
1055 collection = field[len("Collection:") :].strip()
1056 if collection:
1057 for num in citation_nums:
1058 collections[num] = collection
1059 citation_nums = ()
1060 return collections
1062 def _format_source_tagged_hyperlinks(
1063 self,
1064 content: str,
1065 sources: Dict[str, Tuple[str, str]],
1066 collections: Dict[str, str],
1067 ) -> str:
1068 """Replace ``[N]`` with ``[[source-N]](url)``.
1070 ``source`` resolves to (in order): the RAG ``Collection:``
1071 tag for library hits, the short URLClassifier tag for known
1072 academic sources (``arxiv``, ``pubmed``, ...), the cleaned
1073 domain otherwise, or ``local`` for empty/file URLs. ``N`` is
1074 the original global citation number — labels never collide and
1075 the suffix always matches the bibliography ordering.
1077 Args:
1078 content: Document body (sources section already split off).
1079 sources: ``{citation_num: (title, url)}`` parsed from the
1080 sources block.
1081 collections: ``{citation_num: collection_name}`` parsed from
1082 optional ``Collection:`` lines in the sources block
1083 (empty dict when no library/RAG hits are cited). Wins
1084 over URL-derived tags when present for a given citation.
1085 """
1087 # Pre-compute tags so each URL-having source resolves its label
1088 # once, however many [N] / [1, 2, 3] refs cite it.
1089 tag_cache: Dict[str, str] = {
1090 num: self._extract_source_label(
1091 url, collection=collections.get(num)
1092 )
1093 for num, (_, url) in sources.items()
1094 if url
1095 }
1097 def format_link(citation_num, data):
1098 _, url = data
1099 label = tag_cache.get(citation_num)
1100 if label is None:
1101 # Only URL-less sources are absent from tag_cache. Don't
1102 # pass this as dict.get()'s default — that would evaluate
1103 # the expensive lookup eagerly on every cache hit too.
1104 label = self._extract_source_label(
1105 url, collection=collections.get(citation_num)
1106 )
1107 tag = f"{label}-{citation_num}"
1108 # Only emit a hyperlink for http(s) URLs — local/file URLs are
1109 # rendered as plain bracketed tags so the markdown stays clean
1110 # and viewers don't try to navigate to a server-local path.
1111 return (
1112 f"[[{tag}]]({url})"
1113 if self._is_linkable_url(url)
1114 else f"[{tag}]"
1115 )
1117 # Handle comma-separated citations like [1, 2, 3]
1118 content = self._replace_comma_citations(content, sources, format_link)
1120 formatter = self._create_citation_formatter(sources, format_link)
1122 # Handle individual citations
1123 def replace_citation(match):
1124 return (
1125 formatter(match.group(1))
1126 if match.group(1) in sources
1127 else match.group(0)
1128 )
1130 content = self.citation_pattern.sub(replace_citation, content)
1132 # Also handle "Source X" patterns
1133 return self.source_word_pattern.sub(
1134 self._create_source_word_replacer(formatter), content
1135 )
1137 @staticmethod
1138 def _slugify_collection(name: str) -> str:
1139 """Make a user-set collection name safe for inline citations.
1141 Collection names are free-form strings (``"My Papers"``,
1142 ``"team/finance"``). Citations need a compact token that won't
1143 break markdown — strip whitespace, lowercase, replace runs of
1144 non-alphanumeric chars with a single hyphen, trim leading and
1145 trailing hyphens, and fall back to ``"local"`` if the result is
1146 empty. ``-N`` is appended downstream so we strip trailing
1147 hyphens to keep the join clean.
1148 """
1149 slug = re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-")
1150 return slug or "local"
1152 @staticmethod
1153 def _slugify_title(title: str, max_length: int = 32) -> Optional[str]:
1154 """Generate a slug from a document title using python-slugify.
1156 Returns ``None`` when ``title`` is empty / whitespace / pure
1157 punctuation, or when ``slugify`` produces an empty result. The
1158 ``None`` sentinel lets callers distinguish "no meaningful slug"
1159 from "slug happens to be a real word" (e.g. a document literally
1160 titled ``"Doc"`` must yield ``"doc"`` as its label, not fall
1161 through to the citation-number fallback).
1162 """
1163 if not title:
1164 return None
1165 slug = slugify(title, max_length=max_length, separator="-")
1166 return slug if slug else None
1168 def _citation_label(self, citation_num: str, title: str, url: str) -> str:
1169 """Return the inline-citation label for a source.
1171 Order of preference:
1172 1. The URL's domain (``arxiv.org``, ``example.com``) — used by
1173 web citations. Falls through when the URL is relative or
1174 empty, as is common for RAG / library documents.
1175 2. A slugified version of the document title (truncated to
1176 keep labels compact) — gives users a readable label for
1177 local-library hits.
1178 3. The citation number itself — last-resort guarantee the
1179 label is never empty (an empty label produces ``[[]]``
1180 which renders as an uninformative ``[]`` link).
1181 """
1182 domain = self._extract_domain(url) if url else ""
1183 if domain:
1184 return domain
1185 slug = self._slugify_title(title)
1186 if slug:
1187 return slug
1188 return str(citation_num)
1190 @staticmethod
1191 def _is_linkable_url(url: str) -> bool:
1192 """Return True iff ``url`` is a http(s) URL safe to wrap in a
1193 markdown hyperlink. Empty strings and file:// / local: schemes
1194 are not linkable.
1196 Built on ``_safe_link_destination`` so the scheme and control-character
1197 rules live in one place; the only thing added here is that a scheme has
1198 to be PRESENT. SOURCE_TAGGED uses that to decide whether to render a
1199 link at all, which is why it, alone among the modes, does not link a
1200 relative URL.
1201 """
1202 if not _safe_link_destination(url):
1203 return False
1204 try:
1205 return urlsplit(url.strip()).scheme in _LINKABLE_URL_SCHEMES
1206 except ValueError:
1207 return False
1209 def _extract_source_label(
1210 self, url: str, collection: str | None = None
1211 ) -> str:
1212 """Return a short source tag for ``url``.
1214 Resolution order:
1215 1. ``collection`` (when supplied) wins outright — RAG / library
1216 hits surface their collection name as the citation tag
1217 (``mypapers``, ``personal-notes``, ...). The renderer in
1218 ``utilities/search_utilities.format_links_to_markdown``
1219 emits a ``Collection:`` line per source for library results,
1220 which the formatter parses back into this argument.
1221 2. Empty URL or non-http(s) scheme (``file://``, ``local:``, ...) →
1222 ``"local"``. Uniform fallback when no collection name is
1223 available.
1224 3. ``URLClassifier`` matches a known academic source → use the
1225 enum value (``arxiv``, ``pubmed``, ``pmc``, ``biorxiv``,
1226 ``medrxiv``, ``semantic_scholar``, ``doi``).
1227 4. Otherwise → fall back to ``_extract_domain`` (e.g.
1228 ``arxiv.org``, ``nytimes.com``).
1229 """
1230 if collection:
1231 return self._slugify_collection(collection)
1232 if not url:
1233 return "local"
1234 try:
1235 parsed = urlparse(url)
1236 except (ValueError, AttributeError):
1237 return "local"
1238 scheme = (parsed.scheme or "").lower()
1239 if scheme not in ("http", "https"):
1240 return "local"
1242 url_type = URLClassifier.classify(url)
1243 # Generic HTML/PDF/INVALID → fall back to domain. Everything else
1244 # is a known academic source whose enum value is the short tag.
1245 if url_type in (URLType.HTML, URLType.PDF, URLType.INVALID):
1246 return self._extract_domain(url)
1247 return str(url_type.value)
1249 def _extract_domain(self, url: str) -> str:
1250 """Extract domain name from URL."""
1251 try:
1252 # The authority as browsers and requests read it: they end it at
1253 # a backslash, where urllib reads on, so
1254 # https://evil.example\@arxiv.org/ reaches evil.example.
1255 authority = urlparse("//" + urlparse(url).netloc.split("\\", 1)[0])
1256 # Without any userinfo: https://arxiv.org@evil.example/ reaches
1257 # evil.example, and https://user:token@example.com/ would put
1258 # the token in the label.
1259 domain = authority.netloc.rpartition("@")[2]
1260 # Remove www. prefix if present
1261 if domain.startswith("www."):
1262 domain = domain[4:]
1263 # Keep known domains as-is
1264 known_domains = {
1265 "arxiv.org": "arxiv.org",
1266 "github.com": "github.com",
1267 "reddit.com": "reddit.com",
1268 "youtube.com": "youtube.com",
1269 "pypi.org": "pypi.org",
1270 "milvus.io": "milvus.io",
1271 "medium.com": "medium.com",
1272 }
1274 # The host itself or a subdomain of it. A substring test on the
1275 # netloc also labelled arxiv.org.evil.example as arxiv.org. The
1276 # root dot of a fully qualified name (arxiv.org.) is dropped, as
1277 # utilities/arxiv.py does, since that host is still arxiv.org.
1278 # A port that is not a number hides the rest of the authority
1279 # from `hostname` (arxiv.org:443.evil.example), and requests
1280 # refuses it, so it earns no known label.
1281 try:
1282 _ = authority.port
1283 host = (authority.hostname or "").rstrip(".")
1284 except ValueError:
1285 host = ""
1286 for known, display in known_domains.items():
1287 if host == known or host.endswith("." + known):
1288 return display
1290 # Preserve valid ports on other domains, as the existing
1291 # citation-label contract requires. For an invalid authority,
1292 # leave out a port that the check above refused (arxiv.org:99999);
1293 # a colon followed by more labels (arxiv.org:443.evil.example)
1294 # is still part of the untrusted name.
1295 name, _, port = domain.rpartition(":")
1296 if not host and name and "." not in port and "]" not in port:
1297 domain = name
1298 parts = domain.split(".")
1299 if len(parts) >= 2:
1300 return ".".join(parts[-2:])
1301 return domain
1302 except (ValueError, AttributeError):
1303 return "source"
1306# Whether a ``.qmd`` runs code is decided by more than one reader, and they
1307# disagree with each other. From quarto-cli and knitr:
1308#
1309# languagesWithClasses (src/core/pandoc/pandoc-partition.ts), which picks
1310# the engine, run over the WHOLE file with the /gm flags:
1311# /^[\t >]*```+\s*\{([a-zA-Z][a-zA-Z0-9_.]*)([^}]*)?\}\s*$/gm
1312# breakQuartoMd (src/core/lib/break-quarto-md.ts):
1313# ^\s*(```+)\s*\{([=A-Za-z][=A-Za-z0-9._]*)( *[ ,].*)?\}\s*$
1314# quartoMdToJupyter (src/core/jupyter/jupyter.ts), the Jupyter engine:
1315# "^(\\s*)(```+)\\s*\\{" + kernelspec.language.toLowerCase() +
1316# "( *[ ,].*)?\\}\\s*$"
1317# knitr's md chunk.begin (R/pattern.R):
1318# '^[\t >]*```+\\s*\\{([a-zA-Z0-9_]+( *[ ,].*)?)\\}\\s*$'
1319#
1320# They differ in fence tracking (none for languagesWithClasses and knitr,
1321# none that shields a cell for quartoMdToJupyter), in the prefix they allow
1322# (``>`` for blockquotes), in line splitting (/m treats \r, U+2028 and U+2029
1323# as line ends) and in the meaning of ``\s``. Modelling any one of them left
1324# a cell another one runs. So nothing is modelled: every one of them needs a
1325# run of three or more backticks, optional whitespace, ``{`` and a language
1326# character, and that shape is broken wherever it occurs.
1327#
1328# The whitespace class is the union of Python's ``\s`` (which adds
1329# \x1c-\x1f and \x85), JavaScript's (which adds U+FEFF) and U+180E (a space
1330# before Unicode 6.3). It includes every line break, so a header split
1331# across lines is caught too. A language character is ``=`` or anything
1332# Python's Unicode ``\w`` matches, a superset of the ASCII classes above and
1333# of a locale-dependent ``[a-zA-Z]``. ``{.class}`` and ``{#id}`` (Pandoc
1334# attributes, which no reader runs) are left alone.
1335_ANY_SPACE = r"[\s]"
1336_QUARTO_CELL_HEADER = re.compile(
1337 rf"^(?P<indent>(?:{_ANY_SPACE}|>)*)(?P<fence>`{{3,}}){_ANY_SPACE}*"
1338 r"\{(?P<lang>[=\w][=\w.]*)"
1339 rf"(?P<rest>[^=\w.}}\n\r][^}}\n\r]*)?\}}{_ANY_SPACE}*$"
1340)
1341# The backstop: an opening brace of the same shape anywhere in the text. The
1342# lookbehind starts a match only at the first backtick of a run, which keeps
1343# the scan linear on a long run of backticks.
1344_QUARTO_CELL_BRACE = re.compile(
1345 rf"(?<!`)(?P<head>`{{3,}}{_ANY_SPACE}*)\{{(?=[=\w])"
1346)
1347# Quarto's own line splitting (``lines()`` in src/core/lib/text.ts).
1348_QUARTO_LINE_BREAK = re.compile(r"(\r\n?|\n)")
1351def _defuse_quarto_cells(body: str) -> str:
1352 """Turn executable Quarto cells in report prose into display fences.
1354 Markdown to ``.qmd`` is not format-preserving: a ```` ```{python} ````
1355 fence is inert text in a markdown report and an EXECUTABLE cell in a
1356 Quarto document, which ``quarto render`` — the only reason to ask for this
1357 format — runs. Report bodies carry web-influenced content, so the fence
1358 that arrives need not have been written by the user reading it.
1360 Dropping the braces keeps the code visible and stops it being a cell. A
1361 plain ```` ```python ```` fence is display-only already and is untouched.
1363 This fails closed. Every line shaped like a cell header is demoted, with
1364 any indent or ``>`` prefix, wherever it is: inside another fence, in a
1365 blockquote or in plain prose alike. The demotion removes only the two
1366 braces and the whitespace around them; the language and any options stay,
1367 so ```` ```{python, echo=false} ```` becomes ```` ```python, echo=false
1368 ````. A report that quotes the syntax as an example therefore shows it
1369 without its braces; that is the accepted cost. Any other occurrence of a
1370 fence followed by ``{`` and a language character — inside a line of
1371 prose, split across a line break, or with a ``}`` in its options — loses
1372 only that opening brace, so no text after it is dropped. Neither step
1373 adds or removes a backtick or a line break, so no line changes whether
1374 it opens a Markdown code fence.
1376 One regex test per line and one scan of the whole text: linear.
1377 """
1378 parts = _QUARTO_LINE_BREAK.split(body)
1379 for index in range(0, len(parts), 2):
1380 cell = _QUARTO_CELL_HEADER.match(parts[index])
1381 if cell:
1382 parts[index] = (
1383 f"{cell.group('indent')}{cell.group('fence')}"
1384 f"{cell.group('lang')}{cell.group('rest') or ''}"
1385 )
1386 return _QUARTO_CELL_BRACE.sub(r"\g<head>", "".join(parts))
1389# Quarto and Pandoc read more than cells out of the text before and while
1390# rendering it. Two features are reachable from a report body:
1391#
1392# * Shortcodes. ``{{< env NAME >}}`` or ``{{< meta key >}}`` prints the
1393# renderer's environment or metadata into the output, and
1394# ``{{< include path >}}`` pulls in a file, cells included, before the
1395# engine is chosen.
1396# * YAML metadata blocks. src/core/yaml.ts collects every block in the file
1397# with /gm, after removing HTML comments (/<!--[\W\w]*?-->/gm) and fenced
1398# code, and merges them into the render metadata (engine, filters,
1399# pdf-engine-opt, ...):
1400# /(^)(---[ \t]*[\r\n]+(?![ \t]*[\r\n]+)[\W\w]*?[\r\n]+(?:---|\.\.\.))
1401# ([ \t]*)$/gm
1402# Pandoc's own yaml_metadata_block opens at a ``---`` line in any block
1403# context: after a ``>`` or list prefix, and inside a grid-table cell
1404# (``| --- |``), where the dashes do not end the line. Quarto removes the
1405# front matter before Pandoc reads the file and passes the metadata with
1406# ``--metadata-file`` (src/command/render/pandoc.ts in v1.10.18: the
1407# ``partitionYamlFrontMatter`` call at line 1146 and the push at 1351), so
1408# the reader format below switches that extension off and Pandoc reads no
1409# metadata block from the body at all.
1410#
1411# Shortcodes are the hard part, because Quarto decodes text in many places
1412# before it looks for them (quarto-cli v1.10.18): Pandoc entities and
1413# backslash escapes in attribute values, a callout title or a caption
1414# attribute read again as a whole document (``string_to_quarto_ast_*``),
1415# ``qmd``/``qmd-base64`` attributes on a Span or Div, raw blocks in the
1416# ``pandoc-native``, ``pandoc-json`` or ``pandoc-reader-<any reader>``
1417# formats, ``\QuartoMarkdownBase64{}`` in raw LaTeX, and raw HTML whose
1418# attributes and tables Pandoc's HTML reader parses. Breaking each
1419# spelling in turn has failed repeatedly, so the exported file switches
1420# those readers off instead. Its front matter sets
1421# ``from: _QUARTO_READER_FORMAT``: Pandoc's markdown without raw HTML, raw
1422# TeX, raw attributes, fenced or native divs and spans, attribute blocks
1423# on headers, links, images, code and fences, YAML metadata blocks or a
1424# ``%`` title block.
1425# Quarto passes ``from`` to its reader (``user-defined-from`` in
1426# src/command/render/pandoc.ts, used by readqmd.lua for the document and
1427# for every string read again through ``string_to_quarto_ast_*``), so the
1428# body yields no attribute, no raw block or inline, no Div or Span and no
1429# metadata, and every decoder above is unreachable. The front matter itself
1430# is not read with that format: Quarto passes it to Pandoc with
1431# ``--metadata-file`` (pandoc.ts:1351), and Pandoc parses a metadata file
1432# with all of its default markdown extensions whenever the reader is not
1433# literally ``markdown`` or ``commonmark`` (``readerOptsMeta`` in Pandoc's
1434# App.hs). Quarto's reader is a Lua custom reader
1435# (filters/qmd-reader.lua), so raw HTML, a raw ``{=pandoc-native}`` inline
1436# or a ``[...]{qmd-base64=...}`` span in the title would reach the
1437# decoders above. ``_quarto_title_markdown`` therefore escapes the title
1438# so that Pandoc reads it as plain text, whatever the extensions.
1439# The same reader format applies to the markup Quarto itself
1440# appends to the body before Pandoc reads it (pandoc.ts:1275-1283): the
1441# ``:::{#quarto-meta-markdown .hidden}`` envelope of ``[...]{.hidden ...}``
1442# spans in a website, blog or book project (website.ts:328-330,
1443# website-meta.ts:588, markdown-pipeline.ts:99-121), and the same kind of
1444# envelope for revealjs (format-reveal.ts:304) or a document that sets
1445# ``math:`` (pandoc.ts:627-659), as well as the ``keep-source`` block
1446# (codetools.ts:72-89). The exported file sets none of these and is meant
1447# to be rendered on its own. Rendered inside such a project or with those
1448# options, that markup shows as text at the end of the page and what it
1449# feeds (the rendered page and site titles and descriptions) is lost. No
1450# reader format can keep Quarto's markup and drop the report's: both are
1451# read in one pass, and the Divs and Spans that markup needs are what the
1452# report would use to reach the decoders above.
1453# What still decodes body text is one fixed chain:
1454#
1455# 1. the raw file. lpegshortcode.lua opens a shortcode at a literal
1456# ``{{<`` (escaped ``{{{<`` and ``{{</*``), and parse-shortcode.ts,
1457# which expands includes, at a literal ``{{< ``. ``wrap_lpeg_match``
1458# then restores ``cf5733e5-0370-4aae-8689-61bad1dd9ec0&#x<hex>;`` to
1459# the character it encodes, after matching but before Pandoc reads
1460# the text.
1461# 2. Pandoc, once: an entity (``&`` and non-space characters up to the
1462# first ``;``, looked up as ``&name;``, ``{`` or ``{``) or a
1463# backslash escape becomes its character, in text, link targets and
1464# image sources. Code, code blocks and math are not decoded.
1465# 3. readqmd.lua hex-decodes
1466# ``b58fc729-690b-4000-b19f-365a4093b2ff;<hex>;`` in text, code, math,
1467# code blocks, link targets and image sources and reads the result as
1468# a shortcode, and customnodes/shortcodes.lua runs the ``{{<`` parser
1469# again on code, code blocks, math, link targets and image sources.
1470#
1471# Those two UUIDs are the only shortcode encodings in quarto-cli's filters
1472# (a sweep of every UUID in src/resources finds the others naming ids,
1473# data URIs, LaTeX placeholders, the AST vault or a Confluence macro).
1474# A few Quarto filters still read text with Pandoc's own reader, every
1475# extension on: a ``{...}`` ending a table caption (common/tables.lua),
1476# ``{#eq-...}`` after display math (crossref/equations.lua) and a
1477# ```` ```quarto-title-block ```` fence (quarto-post/book.lua, after
1478# shortcodes have run). In v1.10.18 none of them reaches a shortcode
1479# parser or the ``.qmd`` reader.
1480# Each step is broken by
1481# inserting a zero-width space (U+200B), which no reader treats as
1482# whitespace and no decoding removes:
1483#
1484# A. ``b58fc729...`` and ``cf5733e5...`` are broken after their first
1485# letter, and so is ``quarto-shortcode__``, the class ast/parse.lua
1486# turns into a shortcode (no Span can carry it any more; this is kept
1487# as a second line). A letter can be spelled only as itself or as an
1488# entity (rule B); ``-`` and ``_`` also with backslashes before them.
1489# B. An ``&`` gets the space after it if Pandoc would decode it (step 2)
1490# to ``{``, ``-``, ``_`` or an ASCII letter or digit (the characters
1491# of an opener or a sentinel), or if it starts ``&#`` and anything but
1492# a plain decimal or hex reference of at most 8 digits up to the
1493# ``;``. Pandoc decodes only once, so ``&``, ``<`` or `` ``
1494# stay as they are, and so does every ``&`` that starts no entity: one
1495# that no ``;`` closes before a space, as in a URL query
1496# (``?q=a&hl=en``) or C code (``&x``), or one whose name is not an
1497# HTML5 entity.
1498# C. Two ``{``, with any backslashes before the second, followed after any
1499# whitespace or backslashes by ``<``, ``%`` or ``&`` (which may decode
1500# to either or to whitespace) get the space after the first. No step
1501# can then put ``{{`` before a ``<`` or ``%``. ``{{\partial`` in math
1502# is left alone.
1503# HTML comment openers ``<!--`` get the zero-width space between the
1504# dashes, so yaml.ts cannot remove a comment to join text into a new line
1505# start. Three dashes ending a line become four when everything before
1506# them on the line is whitespace and block-prefix characters (``>``,
1507# ``:``, ``~``, ``|``) or footnote labels (``[^1]:``): ``----`` is the
1508# same thematic break or setext underline to Markdown but opens no YAML
1509# block for Quarto's yaml.ts, which needs the dashes at the very start of
1510# a line. Any other text before them on the line leaves them alone, so
1511# prose ending in `` ---`` keeps its em dash and a URL ending in ``---``
1512# keeps its dashes. Pandoc reads no body metadata block with the reader
1513# format below, wherever the dashes are; the block prefixes are a second
1514# line against that. Nor does it read a ``%`` title block: Quarto starts the
1515# body it hands to Pandoc with a line break (``partitionYamlFrontMatter``
1516# in src/core/yaml.ts), which already keeps one from opening, and the
1517# reader format switches the extension off as well.
1518_QUARTO_READER_FORMAT = "markdown" + "".join(
1519 f"-{extension}"
1520 for extension in (
1521 "raw_html",
1522 "raw_tex",
1523 "raw_attribute",
1524 "native_divs",
1525 "native_spans",
1526 "fenced_divs",
1527 "bracketed_spans",
1528 "header_attributes",
1529 "link_attributes",
1530 "inline_code_attributes",
1531 "fenced_code_attributes",
1532 "yaml_metadata_block",
1533 "pandoc_title_block",
1534 )
1535)
1536_ZERO_WIDTH_SPACE = "\u200b"
1537_HTML_COMMENT_OPEN = re.compile(r"<!-(?=-)")
1538# Matched against one line at a time; group 1 is the whole prefix. Linear:
1539# anchored, and each prefix unit starts with a character no other unit
1540# starts with.
1541_YAML_BLOCK_DELIMITER = re.compile(
1542 rf"\A((?:[\s\ufeff\u180e>:~|]|\[\^[^\]]+\]:)*)---(?={_ANY_SPACE}*\Z)"
1543)
1544# Every line break any reader uses: Python's ``str.splitlines`` set, which
1545# includes JavaScript's /m line terminators (\n, \r, U+2028, U+2029).
1546_ANY_LINE_BREAK = re.compile(r"(\r\n|[\n\r\v\f\x1c-\x1e\x85\u2028\u2029])")
1548# Pandoc's ``characterReference``: ``&``, then ``many1Till nonspaceChar
1549# (char ';')`` where a space is one of these four characters.
1550_PANDOC_SPACE = re.compile(r"[ \t\n\r]")
1551_SHORTCODE_CHARACTERS = frozenset(
1552 "{-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
1553)
1554_NUMERIC_REFERENCE = re.compile(r"#(?:([0-9]{1,8})|[xX]([0-9a-fA-F]{1,8}))")
1555# Longer than any HTML5 entity name (31 characters) or plain numeric
1556# reference (``#x`` and 8 digits), so never sliced out of the text.
1557_LONGEST_REFERENCE = 40
1558_QUARTO_SHORTCODE_BRACES = re.compile(
1559 rf"\{{(?=\\*\{{(?:{_ANY_SPACE}|\\)*[<%&])"
1560)
1561_QUARTO_SHORTCODE_TOKENS = re.compile(
1562 r"(?<=b)(?=58fc729\\*-690b\\*-4000\\*-b19f\\*-365a4093b2ff)"
1563 r"|(?<=c)(?=f5733e5\\*-0370\\*-4aae\\*-8689\\*-61bad1dd9ec0)"
1564 r"|(?<=q)(?=uarto\\*-shortcode\\*_\\*_)",
1565 re.IGNORECASE,
1566)
1569def _reference_is_hazardous(reference: str) -> bool:
1570 """True if Pandoc decodes ``&<reference>;`` to a character rule B breaks.
1572 *reference* is what Pandoc reads between the ``&`` and the ``;``. A
1573 numeric one is hazardous unless it is a plain decimal or hex number of
1574 a harmless character; a name is hazardous if it is an HTML5 entity for
1575 a character that can spell a shortcode or a sentinel. Any other name
1576 is no entity, so Pandoc keeps it as text.
1577 """
1578 if reference.startswith("#"):
1579 number = _NUMERIC_REFERENCE.fullmatch(reference)
1580 if number is None:
1581 return True
1582 decimal, hexadecimal = number.groups()
1583 code = int(decimal, 10) if decimal else int(hexadecimal, 16)
1584 if not 0 < code <= 0x10FFFF: 1584 ↛ 1585line 1584 didn't jump to line 1585 because the condition on line 1584 was never true
1585 return True
1586 return chr(code) in _SHORTCODE_CHARACTERS
1587 value = html5.get(reference + ";")
1588 return value is not None and bool(_SHORTCODE_CHARACTERS.intersection(value))
1591def _break_entities(text: str) -> str:
1592 """Rule B: no entity Pandoc decodes can spell a shortcode or sentinel.
1594 The ``;`` and the space after each ``&`` are found by searches that
1595 only move forward, so the scan is linear.
1596 """
1597 pieces = []
1598 position = 0
1599 semicolon = space = -1
1600 start = text.find("&")
1601 while start != -1:
1602 if semicolon < start + 2:
1603 semicolon = text.find(";", start + 2)
1604 if semicolon == -1:
1605 break
1606 if space <= start:
1607 found = _PANDOC_SPACE.search(text, start + 1)
1608 space = found.start() if found else len(text)
1609 if space > semicolon and (
1610 text[start + 1] == "#"
1611 if semicolon - start > _LONGEST_REFERENCE
1612 else _reference_is_hazardous(text[start + 1 : semicolon])
1613 ):
1614 pieces.append(text[position : start + 1])
1615 pieces.append(_ZERO_WIDTH_SPACE)
1616 position = start + 1
1617 start = text.find("&", start + 1)
1618 pieces.append(text[position:])
1619 return "".join(pieces)
1622def _defuse_quarto_text(text: str) -> str:
1623 """Break every shortcode and HTML-comment opener in *text*.
1625 Rules A-C above, for body text read with ``_QUARTO_READER_FORMAT``.
1626 Used on the report title too, before ``_quarto_title_markdown``
1627 escapes it (the title is not read with that format): Quarto expands
1628 shortcodes in metadata, and a comment opened in the title could be
1629 closed in the body. Only zero-width spaces are inserted; removing them
1630 gives back *text*. Linear: a constant number of scans.
1631 """
1632 text = _break_entities(text)
1633 text = _HTML_COMMENT_OPEN.sub("<!-" + _ZERO_WIDTH_SPACE, text)
1634 text = _QUARTO_SHORTCODE_BRACES.sub("{" + _ZERO_WIDTH_SPACE, text)
1635 return _QUARTO_SHORTCODE_TOKENS.sub(_ZERO_WIDTH_SPACE, text)
1638def _defuse_quarto_body(body: str) -> str:
1639 """Neutralise the cells, shortcodes and metadata blocks in a report body.
1641 Cells are demoted (``_defuse_quarto_cells``), shortcodes and HTML
1642 comments are broken (``_defuse_quarto_text``) and no line can open a
1643 YAML metadata block for Quarto (three dashes ending a line after
1644 nothing but whitespace, ``>``, ``:``, ``~``, ``|`` and footnote labels
1645 gain a fourth; prose ending in `` ---`` is left alone; Pandoc's own
1646 metadata blocks, which also open inside a grid-table cell, are
1647 switched off by ``_QUARTO_READER_FORMAT``). The exported file's
1648 metadata is then only the exporter's own front matter, which names no
1649 engine; no cell header is left and no file can be included, so
1650 Quarto's engine detection finds no language and picks its markdown
1651 engine, which runs no code, inline `` `r ...` `` and
1652 `` `{python} ...` `` expressions included. That holds for the file
1653 rendered on its own: an engine forced from outside it (a project
1654 ``_quarto.yml``, ``quarto render -M engine:knitr`` or ``knitr::knit``)
1655 would still evaluate those inline expressions, and a reader format
1656 that overrides the file's ``from`` from outside would turn the readers
1657 listed above back on.
1659 The steps only insert characters, or remove a cell's opening brace or a
1660 cell header's braces and the whitespace around them. None of that can
1661 create an opener another step breaks, so the result is a fixed point.
1662 Linear: a constant number of scans.
1663 """
1664 body = _defuse_quarto_cells(_defuse_quarto_text(body))
1665 parts = _ANY_LINE_BREAK.split(body)
1666 for index in range(0, len(parts), 2):
1667 parts[index] = _YAML_BLOCK_DELIMITER.sub(r"\1----", parts[index])
1668 return "".join(parts)
1671# Every ASCII punctuation character. Pandoc's markdown reads a backslash
1672# before any of them as that character (``all_symbols_escapable``).
1673_ASCII_PUNCTUATION = re.compile(r"([!-/:-@\[-`{-~])")
1674# Runs of what Pandoc's markdown reader treats as spaces and line endings.
1675_PANDOC_SPACE_RUN = re.compile(r"[ \t\n\r]+")
1678def _quarto_title_markdown(title: str) -> str:
1679 """The Markdown source for *title* in the exported front matter.
1681 Pandoc reads the front matter with all of its default markdown
1682 extensions, not with ``_QUARTO_READER_FORMAT`` (see the comment above
1683 it), so the title gets its own treatment. Runs of spaces, tabs and
1684 line breaks become one space and the ends are trimmed, so no line can
1685 start a block such as an indented code block; shortcodes and comment
1686 openers are broken (``_defuse_quarto_text``); and a backslash goes
1687 before every ASCII punctuation character. Pandoc then reads the title
1688 as ``Str`` and ``Space`` nodes only: no raw HTML or raw ``{=format}``
1689 inline, no span or attribute, no markup, entity, math or citation.
1690 The title renders as the same characters, with its whitespace
1691 collapsed; Markdown formatting, entities and typographic quotes or
1692 dashes in it are shown as written. Linear.
1693 """
1694 title = _PANDOC_SPACE_RUN.sub(" ", title).strip(" ")
1695 return _ASCII_PUNCTUATION.sub(r"\\\1", _defuse_quarto_text(title))
1698# Characters that must be escaped inside a YAML double-quoted scalar: the
1699# quote and backslash the double-quoted form itself requires, plus the
1700# control/line-break range ``is_line_breaking_char()`` above defines once
1701# for every producer in this module (C0, DEL, C1, and the Unicode line
1702# separators U+2028/U+2029).
1703_YAML_DQ_SPECIALS_RE = re.compile(r'["\\\x00-\x1f\x7f-\x9f\u2028\u2029]')
1706def _yaml_double_quote(value: str) -> str:
1707 r"""Render *value* as a YAML double-quoted scalar.
1709 Report titles derive from web-influenced content, so a bare
1710 interpolation into ``title: "..."`` lets a quote end the scalar
1711 early (front matter stops parsing) or a newline inject further
1712 front-matter keys. Escaping to YAML double-quoted rules keeps the
1713 value a single scalar that parses back exactly.
1714 """
1716 def _escape(match: re.Match) -> str:
1717 ch = match.group()
1718 if ch == '"':
1719 return '\\"'
1720 if ch == "\\":
1721 return "\\\\"
1722 if ch == "\n": 1722 ↛ 1723line 1722 didn't jump to line 1723 because the condition on line 1722 was never true
1723 return "\\n"
1724 if ch == "\r": 1724 ↛ 1725line 1724 didn't jump to line 1725 because the condition on line 1724 was never true
1725 return "\\r"
1726 if ch == "\t": 1726 ↛ 1727line 1726 didn't jump to line 1727 because the condition on line 1726 was never true
1727 return "\\t"
1728 code = ord(ch)
1729 return f"\\x{code:02x}" if code <= 0xFF else f"\\u{code:04x}"
1731 return '"' + _YAML_DQ_SPECIALS_RE.sub(_escape, value) + '"'
1734class QuartoExporter:
1735 """Export markdown documents to Quarto (.qmd) format."""
1737 def __init__(self):
1738 # Also match Unicode lenticular brackets 【】 (U+3010 and U+3011) that LLMs sometimes generate
1739 self.citation_pattern = re.compile(
1740 r"(?<![\[【])[\[【](\d+)[\]】](?![\]】])"
1741 )
1742 self.comma_citation_pattern = re.compile(
1743 r"[\[【](\d+(?:\s*,\s*\d+)+)[\]】]" # (#6944) space allowed BEFORE the comma too — a hand-edited `[[2 ,3]]` must stay a group
1744 )
1746 def export_to_quarto(self, content: str, title: str | None = None) -> str:
1747 """
1748 Convert markdown document to Quarto format.
1750 Args:
1751 content: Markdown content
1752 title: Document title (if None, will extract from content)
1754 Returns:
1755 Quarto formatted content
1756 """
1757 # Extract title from markdown if not provided
1758 if not title:
1759 title_match = re.search(r"^#\s+(.+)$", content, re.MULTILINE)
1760 title = title_match.group(1) if title_match else "Research Report"
1762 # Create Quarto YAML header
1763 from datetime import UTC, datetime
1765 current_date = datetime.now(UTC).strftime("%Y-%m-%d")
1766 yaml_header = f"""---
1767title: {_yaml_double_quote(_quarto_title_markdown(title))}
1768author: "Local Deep Research"
1769date: "{current_date}"
1770from: {_QUARTO_READER_FORMAT}
1771format:
1772 html:
1773 toc: true
1774 toc-depth: 3
1775 number-sections: true
1776 pdf:
1777 toc: true
1778 number-sections: true
1779 colorlinks: true
1780bibliography: references.bib
1781csl: apa.csl
1782---
1784"""
1786 # Process content
1787 processed_content = content
1789 # Report math reaches a TeX engine when ``quarto render`` builds the
1790 # declared pdf format, so spans that use commands outside the math
1791 # allowlist are escaped to literal text here, exactly as the LaTeX
1792 # export does. Run before the citation and cell passes so the escaped
1793 # brackets of a rejected span are never mistaken for a citation.
1794 processed_content = _defuse_report_math_for_quarto(processed_content)
1796 # First handle comma-separated citations like [1, 2, 3]
1797 def replace_comma_citations(match):
1798 citation_nums = match.group(1)
1799 # Split by comma and strip whitespace
1800 nums = [num.strip() for num in citation_nums.split(",")]
1801 refs = [f"@ref{num}" for num in nums]
1802 return f"[{', '.join(refs)}]"
1804 processed_content = self.comma_citation_pattern.sub(
1805 replace_comma_citations, processed_content
1806 )
1808 # Then convert individual citations to Quarto format [@citation]
1809 def replace_citation(match):
1810 citation_num = match.group(1)
1811 return f"[@ref{citation_num}]"
1813 processed_content = self.citation_pattern.sub(
1814 replace_citation, processed_content
1815 )
1817 # The report generator's sources sentinel is an HTML comment, which
1818 # the reader format would show as text (raw HTML is off) and
1819 # _defuse_quarto_body would break. Drop the one the bibliography is
1820 # scoped by (the last one on its own line); a quoted one elsewhere
1821 # is report text and is defused like any other comment.
1822 sentinel_start, on_sentinel = find_sources_section(processed_content)
1823 if on_sentinel:
1824 processed_content = (
1825 processed_content[:sentinel_start]
1826 + processed_content[
1827 sentinel_start + len(LDR_APPENDED_SOURCES_SENTINEL) :
1828 ]
1829 )
1831 # Generate bibliography file content. It is shown in a code block,
1832 # which Pandoc does not decode, but readqmd.lua hex-decodes the
1833 # internal shortcode encodings in CodeBlock text too and
1834 # shortcodes.lua then expands the result, so a source title or URL
1835 # carrying one is broken as in the body. Its braces are already
1836 # escaped (``_escape_bibtex``) or rejected (``_safe_bibtex_url``).
1837 bib_content = _QUARTO_SHORTCODE_TOKENS.sub(
1838 _ZERO_WIDTH_SPACE, self._generate_bibliography(content)
1839 )
1841 # Add note about bibliography file. It is a plain paragraph: the
1842 # reader format turns fenced divs off, so a callout would show as
1843 # text.
1844 bibliography_note = (
1845 "\n\n**Bibliography File Required**\n\nThis document requires a `references.bib` file in the same directory with the following content:\n\n```bibtex\n"
1846 + bib_content
1847 + "\n```\n"
1848 )
1850 # The note and its bibtex fence are the exporter's own markup, so
1851 # of the note only the bibliography entries are defused (above).
1852 return (
1853 yaml_header
1854 + _defuse_quarto_body(processed_content)
1855 + bibliography_note
1856 )
1858 def _generate_bibliography(self, content: str) -> str:
1859 """Generate BibTeX bibliography from sources."""
1860 sources_start, _on_sentinel = find_sources_section(content)
1861 # Keep accepting bare source lists for programmatic callers. A full
1862 # report's explicit Sources boundary takes precedence over its body.
1863 sources_content = (
1864 content[sources_start:] if sources_start != -1 else content
1865 )
1866 entries = _collect_bibliography_entries(sources_content)
1868 bibliography = ""
1869 for citation_num in sorted(entries, key=_sort_key):
1870 title, url = entries[citation_num]
1871 safe_title = _escape_bibtex(title)
1872 safe_url = _safe_bibtex_url(url)
1873 bib_entry = f"@misc{{ref{citation_num},\n"
1874 bib_entry += f' title = "{{{safe_title}}}",\n'
1875 if safe_url:
1876 bib_entry += f" url = {{{safe_url}}},\n"
1877 bib_entry += f' howpublished = "\\url{{{safe_url}}}",\n'
1878 bib_entry += f" year = {{{2024}}},\n"
1879 bib_entry += ' note = "Accessed: \\today"\n'
1880 bib_entry += "}\n"
1882 bibliography += bib_entry + "\n"
1884 return bibliography.strip()
1887# One credential check, ``_ris_spells_userinfo``, decides for every RIS
1888# field built from source text: UR, TI, AU and DO are never written with a
1889# value it flags. It also reads each value as serialization writes it
1890# (``_ris_flatten``), so no field is written in a form it flags either. A
1891# source URL goes through ``_safe_ris_url``, which runs the check last; a
1892# title, author or DOI the check flags is replaced by ``_safe_ris_url``'s
1893# userinfo-free URL for it, or dropped (a title, by a placeholder). A
1894# password can hold a name, a year or a DOI, so no author, no year (PY)
1895# and no DOI matched in other text is cut out of a line that carries a
1896# credential (``_ris_line_carries_userinfo``): a line the check flags, or
1897# one in which a value read from it holds a credential when read as a URL
1898# as a whole from where the value starts (``_ris_value_starts_userinfo``).
1899# Nor is a year or DOI taken from a title that its cleaning joins into a
1900# credential URL. PB is derived from the host of the sanitized UR. TY, Y1,
1901# DA, LA and ER are fixed or dated, and ID holds the citation number.
1902# ``_safe_ris_url`` also omits URLs it cannot parse unambiguously; a title,
1903# author or DOI need not be a URL, so such text stays in those fields
1904# unless the check flags it.
1905#
1906# Userinfo is what precedes the last @ of a URL's authority, as the WHATWG
1907# URL Standard parses it. The authority ends at the first ``/``, ``?`` or
1908# ``#``, so a password that holds one of them unencoded is not userinfo,
1909# and the check does not flag it: ``https://user:2024/Secret@host/x`` has
1910# the host ``user``, the port 2024 and no credential. Under any scheme,
1911# and under none (``//host``), an authority follows ``//``. Under
1912# the schemes the WHATWG URL Standard calls special, it also follows the
1913# colon and any run of ``/`` or ``\`` (none included): a browser reads
1914# ``https:user:pass@host`` and ``https:\\user:pass@host`` as credential
1915# URLs. ``file`` is special too, but a file URL carries no userinfo. A
1916# value with no scheme is resolved against a base URL, and against a
1917# special one, such as an ``https`` page, any run of two or more ``/`` or
1918# ``\`` opens an authority as ``//`` does: a browser reads
1919# ``\\user:pass@host``, ``/\user:pass@host`` and ``///user:pass@host``
1920# there as ``https://user:pass@host``.
1921_RIS_SPECIAL_SCHEME = r"(?:https?|ftp|wss?)"
1923# An authority opened anywhere in a value, so a credential URL wrapped in
1924# ``<...>``, a markdown link, quotes, parentheses, backticks or emphasis, or
1925# written inside prose, is still seen; a run of two or more ``/`` or ``\``
1926# opens one wherever it appears, as ``//`` does. In each reading of a
1927# value (``_ris_readings``), a URL inside text ends at whitespace, as a
1928# linkifier ends it, so an e-mail address later in the text is not read
1929# as its userinfo. Neither is what follows whitespace inside a wrapped
1930# URL's authority, as in ``<https://user :pass@host>``; only a value that
1931# is a URL as a whole is read past it. The lookbehind keeps the ``ws:`` of
1932# ``news:`` and the ``ftp:`` of ``sftp:`` from opening an authority.
1933_RIS_EMBEDDED_AUTHORITY_RE = re.compile(
1934 rf"(?:(?<![a-z0-9+.\-]){_RIS_SPECIAL_SCHEME}:[/\\]*|[/\\]{{2,}})"
1935 r"([^\s/?#]*)",
1936 re.IGNORECASE,
1937)
1939# What a value that is a URL as a whole may begin and end with: whitespace,
1940# controls and U+FEFF. RIS serialization turns controls into spaces, and a
1941# reader may trim a field: JavaScript's ``trim()`` removes U+FEFF as well,
1942# which Python's ``strip()`` keeps. Inside a value, U+FEFF is not
1943# whitespace.
1944_RIS_TRIMMED = r"[\s\x00-\x20\x7f-\x9f\ufeff]"
1946# The authority of a value that is a URL as a whole, past any leading
1947# ``_RIS_TRIMMED`` characters. It runs up to the first ``/``, ``?`` or
1948# ``#``, whitespace included, as ``urlsplit`` delimits a netloc after
1949# ``//``. So a title spelling a URL that ``_safe_ris_url`` would strip a
1950# credential from is flagged too, as is ``\\ user:pass@host``, which a
1951# browser reads with the space encoded.
1952#
1953# Unless a base URL of the same scheme makes the value relative, a browser
1954# also reads whitespace after a special scheme's colon, or after the colon
1955# and one ``/`` or ``\``, as part of the authority: to it,
1956# ``https:/ user:pass@host`` and ``https: user:pass@host`` carry
1957# credentials, and so does the title ``HTTP: email me at bob@example.org``,
1958# whose userinfo it reads as `` email me at bob``. After the colon and one
1959# ``/`` or ``\``, the authority runs past whitespace as above; no title
1960# begins that way. So that prose stays a title, an authority opened by the
1961# colon alone, with no ``/`` or ``\`` after it, may begin with whitespace
1962# only in the shape ``_RIS_SPACED_AUTHORITY`` matches. Past that
1963# whitespace must come a token of one or more characters other than
1964# whitespace, ``/``, ``\``, ``?`` and ``#``; optional whitespace; an @;
1965# and a host of characters other than those and @, followed by ``/``,
1966# ``\``, ``?``, ``#`` or nothing but ``_RIS_TRIMMED`` characters, as the
1967# value may begin with them. ``https: user:pass@host/x`` and
1968# ``https: user:pass @host`` are flagged. Three titles are not:
1969# ``HTTP: email me at bob@example.org``, whose would-be userinfo holds
1970# whitespace; ``HTTP: bob@example.org explains``, whose would-be host
1971# does; and ``HTTP: @mention``, which has no token before its @. Every
1972# reading of a value is matched, so a control where this shape takes
1973# whitespace is read as the space serialization writes for it
1974# (``_ris_flatten``). The host is matched possessively: it shares controls
1975# and U+FEFF with ``_RIS_TRIMMED``, and giving them back could only make
1976# the end of the value be scanned again, in time quadratic in its length.
1977_RIS_SPACED_AUTHORITY = "".join(
1978 (
1979 r"\s+[^\s/?#\\]+\s*@[^\s/?#@\\]++(?:[/?#\\]|",
1980 _RIS_TRIMMED,
1981 r"*\Z)",
1982 )
1983)
1984# What opens the authority of a URL as a whole.
1985_RIS_WHOLE_URL_START = (
1986 rf"(?:(?:{_RIS_SPECIAL_SCHEME}:)?[/\\]{{2,}}|[a-z][a-z0-9+.\-]*://"
1987 rf"|{_RIS_SPECIAL_SCHEME}:"
1988 rf"(?:[/\\]|(?!\s)|(?={_RIS_SPACED_AUTHORITY})))"
1989)
1990_RIS_WHOLE_URL_AUTHORITY_RE = re.compile(
1991 rf"{_RIS_TRIMMED}*{_RIS_WHOLE_URL_START}([^/?#]*)",
1992 re.IGNORECASE,
1993)
1995# Where a URL as a whole can start in a value read from a line
1996# (``_ris_line_carries_userinfo``): at the value's start, and after each
1997# comma, ``and`` or ampersand, where the author pattern splits a list into
1998# authors. As at the start of a value, the URL may follow whitespace,
1999# controls and U+FEFF, matched possessively, since none of them can open
2000# an authority. ``_ris_value_starts_userinfo`` finds where each such
2001# authority ends.
2002_RIS_VALUE_URL_START_RE = re.compile(
2003 rf"(?:\A|,|(?<=\s)(?-i:and)(?=\s)|&){_RIS_TRIMMED}*+"
2004 rf"{_RIS_WHOLE_URL_START}",
2005 re.IGNORECASE,
2006)
2007_RIS_AUTHORITY_END_RE = re.compile(r"[/?#]")
2009# Bounds on the readings of one value, which keep the work of checking it
2010# small; a value past any of them counts as flagged. Each layer of encoding
2011# a value carries adds a reading, so an ordinary value has a few, and 64
2012# leaves room for deliberate nesting. A reading can be longer than the
2013# value, since NFKC turns U+FDFA alone into 18 characters, so every
2014# reading is measured, not only the value. RFC 9110 recommends supporting
2015# URIs of 8,000 octets; no title, author, DOI or link needs eight times
2016# that. The total leaves room for the value plus three more readings of
2017# that length.
2018_RIS_MAX_READINGS = 64
2019_RIS_MAX_DECODED_LENGTH = 1 << 16
2020_RIS_MAX_DECODED_TOTAL_LENGTH = 4 * _RIS_MAX_DECODED_LENGTH
2022# The WHATWG URL parser deletes every tab, LF and CR from its input before
2023# it parses it: a browser reads ``ht<TAB>tps:/<LF>/user:pass@host`` as
2024# ``https://user:pass@host``. So the readings of a value include it with
2025# them deleted, or one that an entity or percent-escape spells would hide
2026# a credential URL once decoded: ``	`` splits the scheme of
2027# ``ht	tps:user:pass@host``, and ``%0A`` the slashes of
2028# ``https:/%0A/user:pass@host``.
2029_RIS_URL_DELETED_CHARACTERS = str.maketrans("", "", "\t\n\r")
2032def _ris_flatten(value: str) -> str:
2033 """``value`` as RIS serialization writes it: every character
2034 ``is_line_breaking_char`` matches is turned into a space, so only the
2035 exporter's own line breaks separate RIS fields."""
2036 return "".join(
2037 " " if is_line_breaking_char(char) else char for char in value
2038 )
2041def _ris_readings(value: str) -> set[str] | None:
2042 """``value`` and every form decoding turns it into; None past a bound.
2044 The decodings are HTML character references, NFKC normalization,
2045 percent-decoding, the deletion of every tab, LF and CR, as the WHATWG
2046 URL parser deletes them, and ``_ris_flatten``, which turns every
2047 control character, U+2028 and U+2029 into a space, as serialization
2048 writes a field. Each is applied to every reading until none yields a
2049 new one, so they compose in any order and repeat. An ASCII value
2050 without ``&``, ``%`` or a control character, which none of them can
2051 change, is its only reading. Any other value is measured, and so is
2052 each new reading as a decoding yields it: the result is None once one
2053 of them is longer than ``_RIS_MAX_DECODED_LENGTH``, once they are
2054 longer together than ``_RIS_MAX_DECODED_TOTAL_LENGTH``, or once there
2055 would be more than ``_RIS_MAX_READINGS`` of them.
2056 """
2057 if value.isascii() and not any(
2058 char in "&%" or is_line_breaking_char(char) for char in value
2059 ):
2060 return {value} # None of the decodings can change it.
2061 if len(value) > _RIS_MAX_DECODED_LENGTH:
2062 return None
2063 readings = {value}
2064 total_length = len(value)
2065 pending = [value]
2066 while pending:
2067 form = pending.pop()
2068 for decoded in (
2069 html.unescape(form),
2070 unicodedata.normalize("NFKC", form),
2071 urllib.parse.unquote(form),
2072 form.translate(_RIS_URL_DELETED_CHARACTERS),
2073 _ris_flatten(form),
2074 ):
2075 if decoded in readings:
2076 continue
2077 total_length += len(decoded)
2078 if (
2079 len(readings) == _RIS_MAX_READINGS
2080 or len(decoded) > _RIS_MAX_DECODED_LENGTH
2081 or total_length > _RIS_MAX_DECODED_TOTAL_LENGTH
2082 ):
2083 return None
2084 readings.add(decoded)
2085 pending.append(decoded)
2086 return readings
2089def _ris_spells_userinfo(value: str) -> bool:
2090 """Whether some reading of ``value`` puts a credential in a URL authority.
2092 A credential is userinfo holding anything other than ``@`` signs; an @
2093 outside every authority, as in ``medium.com/@author``, is not one. The
2094 readings are ``value`` and its decoded forms (``_ris_readings``): a
2095 consumer that renders ``@``, decodes ``%40`` or folds a
2096 full-width at sign (U+FF20) reads the @ they spell, and a browser
2097 deletes the tab, LF or CR that ``	``, `` `` or ``%0D`` spells,
2098 as one reading does. Another is the value as serialization writes it
2099 (``_ris_flatten``), so whatever the check passes is written in a form
2100 it passes too. The check is lexical, so no parse error can let a value
2101 through, and a value past the reading bounds counts as flagged.
2102 """
2103 readings = _ris_readings(value)
2104 if readings is None:
2105 return True
2106 for form in readings:
2107 whole = _RIS_WHOLE_URL_AUTHORITY_RE.match(form)
2108 if whole and "@" in whole.group(1).lstrip("@"):
2109 return True
2110 if any(
2111 "@" in match.group(1).lstrip("@")
2112 for match in _RIS_EMBEDDED_AUTHORITY_RE.finditer(form)
2113 ):
2114 return True
2115 return False
2118def _safe_ris_url(raw_url: str | None) -> str:
2119 """Return a source URL ready for a RIS field without its userinfo, or "".
2121 The literal userinfo of a parseable authority is removed and the rest
2122 of the URL kept. The URL is omitted instead when it contains a
2123 line-breaking character; when ``urlsplit`` cannot parse it, or finds no
2124 host or an empty or invalid port in its authority; when its authority
2125 holds a backslash; when it has a scheme but no authority, and the scheme
2126 is http(s) or the URL holds an @; and when ``_ris_spells_userinfo``
2127 flags what would be exported: an entity-, percent- or NFKC-spelled
2128 delimiter left in the authority; a credential URL with no authority
2129 ``urlsplit`` could strip, inside a wrapper such as ``<...>`` or
2130 ``[text](...)``, inside prose, after a scheme-less run of slashes and
2131 backslashes other than ``//``, after U+FEFF, or with its scheme or
2132 slashes split by an encoded tab, LF or CR; or a value past the bounds
2133 on its readings.
2134 """
2135 if not isinstance(raw_url, str): 2135 ↛ 2136line 2135 didn't jump to line 2136 because the condition on line 2135 was never true
2136 return ""
2137 candidate = raw_url.strip()
2138 if not candidate or any(is_line_breaking_char(char) for char in candidate):
2139 # urlsplit silently discards some controls while parsing, but RIS
2140 # consumers may read the original bytes as additional fields.
2141 return ""
2142 try:
2143 parsed = urlsplit(candidate)
2144 if parsed.netloc:
2145 if "\\" in parsed.netloc:
2146 # WHATWG parsers end a special authority at a backslash and
2147 # urlsplit does not, so the two disagree about the host.
2148 return ""
2149 authority = parsed.netloc.rpartition("@")[2]
2150 clean = parsed._replace(netloc=authority)
2151 if not clean.hostname or authority.endswith(":"): 2151 ↛ 2152line 2151 didn't jump to line 2152 because the condition on line 2151 was never true
2152 return ""
2153 _ = clean.port # ValueError for malformed or out-of-range ports.
2154 candidate = urlunsplit(clean)
2155 elif parsed.scheme and (
2156 parsed.scheme.lower() in _LINKABLE_URL_SCHEMES or "@" in candidate
2157 ):
2158 # A scheme followed by user:password@host without // is not a
2159 # parseable authority; it must not survive as a raw RIS field.
2160 return ""
2161 except ValueError:
2162 return ""
2163 if _ris_spells_userinfo(candidate):
2164 return ""
2165 return candidate
2168def _ris_field_text(value: str) -> str:
2169 """``value`` for a free-text RIS field, or ``_safe_ris_url(value)`` when
2170 the credential check flags it ("" when no URL is kept)."""
2171 if _ris_spells_userinfo(value):
2172 return _safe_ris_url(value)
2173 return value
2176def _ris_value_starts_userinfo(value: str) -> bool:
2177 """Whether, in some reading of ``value``, a URL as a whole that starts
2178 at its start, or after a comma, ``and`` or ampersand in it
2179 (``_RIS_VALUE_URL_START_RE``), holds a credential: an @ in its
2180 authority after any leading @ signs, as ``_ris_spells_userinfo`` reads
2181 the authority of a value that is a URL as a whole. The authority runs
2182 to the first ``/``, ``?`` or ``#``. An authority that starts inside
2183 another ends where that one ends, so each is scanned once, and the work
2184 is linear in the length of the readings. A value past the reading
2185 bounds counts as holding one.
2186 """
2187 readings = _ris_readings(value)
2188 if readings is None: 2188 ↛ 2189line 2188 didn't jump to line 2189 because the condition on line 2188 was never true
2189 return True
2190 for form in readings:
2191 authority_end = last_at = -1
2192 for start in _RIS_VALUE_URL_START_RE.finditer(form):
2193 position = start.end()
2194 if position >= authority_end:
2195 end = _RIS_AUTHORITY_END_RE.search(form, position)
2196 authority_end = end.start() if end else len(form)
2197 last_at = form.rfind("@", position, authority_end)
2198 while position < authority_end and form[position] == "@":
2199 position += 1
2200 if last_at >= position:
2201 return True
2202 return False
2205# The labels of the lines whose value ``export_to_ris`` reads whole: the
2206# source URL from a ``URL:`` line and the DOI from a ``DOI:`` line, each
2207# matched as ``export_to_ris`` matches it.
2208_RIS_VALUE_LABEL_RE = re.compile(r"URL:|DOI:")
2211_RIS_SOURCE_VALUE_START = r"(?:\A|URL:|DOI:|by\s++|,|(?<=\s)and(?=\s)|&)"
2212_RIS_SOURCE_VALUE_AUTHORITY_RE = re.compile(
2213 rf"{_RIS_SOURCE_VALUE_START}"
2214 rf"{_RIS_TRIMMED}*+{_RIS_WHOLE_URL_START}([^/?#]*)",
2215 re.IGNORECASE,
2216)
2217# A source has metadata after its URL, so a host need not end the record.
2218# This relaxed tail applies only when userinfo crosses the checked physical
2219# line. A one-line prose title such as "HTTP: bob@example.org explains"
2220# therefore keeps the field guard's documented prose exception.
2221_RIS_SOURCE_SPACED_AUTHORITY_RE = re.compile(
2222 rf"{_RIS_SOURCE_VALUE_START}{_RIS_TRIMMED}*+{_RIS_SPECIAL_SCHEME}:"
2223 r"\s++([^\s/?#\\]+\s*@[^\s/?#@\\]++)(?=[/?#\\]|\s|\Z)",
2224 re.IGNORECASE,
2225)
2228def _ris_multiline_line_carries_userinfo(text: str, position: int) -> bool:
2229 """Check a source line in its unsplit record, before cutting fields.
2231 A raw LF can cut the @ off a credential URL before a title, URL, DOI,
2232 author or year is checked. Track the line's span through the same
2233 decodings as the field guard, so unrelated lines keep their metadata.
2234 Prefix/suffix comparisons conservatively include a decoded token that
2235 crosses a span boundary (for example, ``%4<LF>0`` becoming ``@``).
2236 No raw source text is rewritten or used as a replacement field value.
2237 """
2238 if "\n" not in text:
2239 return False
2240 if text.isascii() and not any(char in text for char in "@%&"):
2241 return False
2242 if len(text) > _RIS_MAX_DECODED_LENGTH: 2242 ↛ 2243line 2242 didn't jump to line 2243 because the condition on line 2242 was never true
2243 return True
2245 start = text.rfind("\n", 0, position) + 1
2246 end = text.find("\n", position)
2247 if end == -1:
2248 end = len(text)
2249 pending = [(text, start, end)]
2250 seen = set(pending)
2251 total_length = len(text)
2252 decoders = (
2253 html.unescape,
2254 lambda value: unicodedata.normalize("NFKC", value),
2255 urllib.parse.unquote,
2256 lambda value: value.translate(_RIS_URL_DELETED_CHARACTERS),
2257 _ris_flatten,
2258 )
2260 def common_edge(left: str, right: str, *, reverse: bool = False) -> int:
2261 if reverse:
2262 left, right = left[::-1], right[::-1]
2263 if left.startswith(right):
2264 return len(right)
2265 return next(
2266 (i for i, (a, b) in enumerate(zip(left, right)) if a != b),
2267 min(len(left), len(right)),
2268 )
2270 while pending:
2271 form, start, end = pending.pop()
2272 for pattern in (
2273 _RIS_EMBEDDED_AUTHORITY_RE,
2274 _RIS_SOURCE_VALUE_AUTHORITY_RE,
2275 _RIS_SOURCE_SPACED_AUTHORITY_RE,
2276 ):
2277 for match in pattern.finditer(form):
2278 authority = match.group(1)
2279 if "@" not in authority.lstrip("@"):
2280 continue
2281 credential_end = match.start(1) + authority.rfind("@")
2282 crosses_line = (
2283 match.start(1) < start or credential_end + 1 >= end
2284 )
2285 if (
2286 start < credential_end
2287 and end > match.start()
2288 and (
2289 pattern is not _RIS_SOURCE_SPACED_AUTHORITY_RE
2290 or crosses_line
2291 )
2292 ):
2293 return True
2295 for decode in decoders:
2296 decoded = decode(form)
2297 if decoded == form:
2298 continue
2299 if len(decoded) > _RIS_MAX_DECODED_LENGTH: 2299 ↛ 2300line 2299 didn't jump to line 2300 because the condition on line 2299 was never true
2300 return True
2301 before = common_edge(decoded, decode(form[:start]))
2302 after = len(decoded) - common_edge(
2303 decoded, decode(form[end:]), reverse=True
2304 )
2305 state = (decoded, min(before, after), max(before, after))
2306 if state in seen:
2307 continue
2308 total_length += len(decoded)
2309 if ( 2309 ↛ 2314line 2309 didn't jump to line 2314 because the condition on line 2309 was never true
2310 len(seen) == _RIS_MAX_READINGS
2311 or len(decoded) > _RIS_MAX_DECODED_LENGTH
2312 or total_length > _RIS_MAX_DECODED_TOTAL_LENGTH
2313 ):
2314 return True
2315 seen.add(state)
2316 pending.append(state)
2317 return False
2320def _ris_line_carries_userinfo(
2321 text: str, position: int, value_starts: tuple[int, ...] = ()
2322) -> bool:
2323 """Whether the line of ``text`` that holds ``position`` carries a
2324 credential, so that no field may be cut out of it.
2326 The credential check reads the line itself, and each value read from
2327 the line from where it starts to the end of the line
2328 (``_ris_value_starts_userinfo``): the value after a ``URL:`` or ``DOI:``
2329 label that begins the line, which ``export_to_ris`` reads as the link
2330 or the DOI, and the values that start in the line at ``value_starts``,
2331 where the author and DOI patterns begin to read. In the line, a URL
2332 after a label or a word ends at whitespace, as in other text, and the
2333 author and DOI patterns end a value at a full stop, a comma, ``and``,
2334 an ampersand or whitespace, past any of which a URL as a whole can go
2335 on. So ``URL: https://user:by Smith@host/`` is not flagged as a line,
2336 but its value is, with the password ``by Smith``, and the author
2337 pattern would cut ``Smith@host/`` out of the line.
2338 """
2339 start = text.rfind("\n", 0, position) + 1
2340 end = text.find("\n", position)
2341 if end == -1:
2342 end = len(text)
2343 if _ris_spells_userinfo(text[start:end]):
2344 return True
2345 starts = [value for value in value_starts if start <= value <= end]
2346 stripped = text[start:end].lstrip()
2347 label = _RIS_VALUE_LABEL_RE.match(stripped)
2348 if label:
2349 starts.append(end - len(stripped) + label.end())
2350 return any(
2351 _ris_value_starts_userinfo(text[value:end]) for value in starts
2352 ) or _ris_multiline_line_carries_userinfo(text, position)
2355# Metadata clauses cut from a RIS title, applied in this order. Each one
2356# starts at its keyword: a leading ``\s*`` would retry from every position
2357# of a whitespace run before failing, which is quadratic in the run length.
2358# ``_cut_title_clause`` removes the whitespace before the keyword instead.
2359_RIS_TITLE_CLAUSES = (
2360 re.compile(r"DOI:\s*[^\s]+", re.IGNORECASE),
2361 re.compile(r"Published in.*", re.IGNORECASE),
2362 re.compile(r"Volume.*", re.IGNORECASE),
2363 re.compile(r"Pages.*", re.IGNORECASE),
2364)
2367def _cut_title_clause(clause: re.Pattern, text: str) -> str:
2368 r"""Remove every *clause* match together with the whitespace before it.
2370 Gives the same result as ``re.sub(r"\s*" + clause.pattern, "", text)``
2371 in linear time. ``rstrip()`` and ``\s`` treat the same characters as
2372 whitespace. The strip stops at the end of the previous match, which is
2373 where the leading ``\s*`` of the next match could start at the earliest.
2374 """
2375 pieces = []
2376 position = 0
2377 for match in clause.finditer(text):
2378 pieces.append(text[position : match.start()].rstrip())
2379 position = match.end()
2380 if not pieces:
2381 return text
2382 pieces.append(text[position:])
2383 return "".join(pieces)
2386class RISExporter:
2387 """Export references to RIS format for reference managers like Zotero."""
2389 def __init__(self):
2390 self.sources_pattern = re.compile(
2391 r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
2392 re.MULTILINE,
2393 )
2395 def export_to_ris(self, content: str) -> str:
2396 """
2397 Extract references from markdown and convert to RIS format.
2399 Args:
2400 content: Markdown content with sources
2402 Returns:
2403 RIS formatted references
2404 """
2405 # Find sources section
2406 sources_start, _on_sentinel = find_sources_section(content)
2407 if sources_start == -1:
2408 return ""
2410 # Find the end of the first sources section (before any other major section)
2411 sources_content = content[sources_start:]
2413 # Look for the next major section to avoid duplicates
2414 next_section_markers = [
2415 "\n## ALL SOURCES",
2416 "\n### ALL SOURCES",
2417 "\n## Research Metrics",
2418 "\n### Research Metrics",
2419 "\n## SEARCH QUESTIONS",
2420 "\n### SEARCH QUESTIONS",
2421 "\n## DETAILED FINDINGS",
2422 "\n### DETAILED FINDINGS",
2423 "\n---", # Horizontal rule often separates sections
2424 ]
2426 sources_end = len(sources_content)
2427 for marker in next_section_markers:
2428 pos = sources_content.find(marker)
2429 if pos != -1 and pos < sources_end:
2430 sources_end = pos
2432 sources_content = sources_content[:sources_end]
2434 # Parse sources and generate RIS entries
2435 ris_entries = {}
2437 # Split sources into individual entries
2438 import re
2440 # Pattern to match each source entry. The opener is GROUP-AWARE
2441 # (``\d+(?:,\s*\d+)*``, the spelling ``_BIB_SOURCES_PATTERN`` above
2442 # uses for Sources lines; the inline ``comma_citation_pattern``,
2443 # ``\d+(?:\s*,\s*\d+)+``, also accepts whitespace before a comma
2444 # since #6944, while ``format_links_to_markdown`` joins a merged
2445 # opener's indices with ``", "``):
2446 # ``format_links_to_markdown`` collapses every citation of one source
2447 # onto a single merged ``[1, 3] Title`` line, which is routine for
2448 # library documents since per-document keying (#5685). A single-index
2449 # ``(\d+)`` capture did not merely drop such a line — it MIS-CITED
2450 # (#5687), because the lookahead did not terminate at a merged opener
2451 # either, so the PRECEDING entry swallowed the merged line and the
2452 # last-wins ``URL:`` scan below then handed that entry the merged
2453 # line's URL: one source's title against another source's link. Hence
2454 # the group spelling appears TWICE — once in the capture, once in the
2455 # lookahead — and the loop emits one record PER INDEX so that every
2456 # ``[N]`` in the body resolves to a reference of its own.
2457 #
2458 # Four properties here are deliberate; keep them when editing:
2459 # 1. This parser stays bounded to the FIRST Sources section by the
2460 # ``next_section_markers`` scan above. Do not swap in the
2461 # module-level ``_collect_bibliography_entries`` — that helper
2462 # is deliberately not section-scoped, which is why #5687 notes
2463 # RIS is not a mechanical swap even though BibTeX, Quarto and
2464 # LaTeX were fixed by adopting it.
2465 # 2. Lenticular "【N】" openers/closers are accepted alongside
2466 # ASCII "[N]": the inline citation patterns in this file already
2467 # handle them (some LLMs emit them), so the source-list parser
2468 # must stay consistent or it would silently drop those entries.
2469 # 3. Entries are split by LOOKAHEAD under ``DOTALL`` so continuation
2470 # lines (``URL:``, ``DOI:``, ``Published in``) stay attached to
2471 # the entry they belong to; a line-anchored ``$`` split would
2472 # strip them.
2473 # 4. ``self.sources_pattern`` above is group-aware but UNUSED, and
2474 # is not this fix — it is single-line and section-unaware, so
2475 # wiring it in would lose properties 1 and 3.
2476 source_entry_pattern = re.compile(
2477 r"^[\[【](\d+(?:,\s*\d+)*)[\]】]\s*(.+?)"
2478 r"(?=^[\[【]\d+(?:,\s*\d+)*[\]】]|\Z)",
2479 re.MULTILINE | re.DOTALL,
2480 )
2482 for match in source_entry_pattern.finditer(sources_content):
2483 citation_group = match.group(1)
2484 entry_text = match.group(2).strip()
2486 lines = entry_text.split("\n")
2488 # Extract URL, DOI, and other metadata from subsequent lines
2489 url = ""
2490 metadata = {}
2491 value_positions = {}
2492 line_start = len(lines[0]) + 1
2493 for line in lines[1:]:
2494 position = line_start
2495 line_start += len(line) + 1
2496 line = line.strip()
2497 if line.startswith("URL:"):
2498 url = line[4:].strip()
2499 value_positions["url"] = position
2500 elif line.startswith("DOI:"):
2501 metadata["doi"] = line[4:].strip()
2502 value_positions["doi"] = position
2503 elif line.startswith("Published in"):
2504 metadata["journal"] = line[12:].strip()
2505 # Add more metadata parsing as needed
2506 elif line:
2507 # Store other lines as additional metadata
2508 if "additional" not in metadata:
2509 metadata["additional"] = []
2510 additional = metadata["additional"]
2511 if isinstance(additional, list): 2511 ↛ 2493line 2511 didn't jump to line 2493 because the condition on line 2511 was always true
2512 additional.append(line)
2514 # A line split may already have removed the credential's @.
2515 # Check the selected values against the unsplit source before
2516 # the ordinary whole-field sanitizer handles parseable URLs.
2517 if (
2518 url
2519 and not _ris_spells_userinfo(url)
2520 and _ris_multiline_line_carries_userinfo(
2521 entry_text, value_positions["url"]
2522 )
2523 ):
2524 url = ""
2525 if (
2526 metadata.get("doi")
2527 and not _ris_spells_userinfo(metadata["doi"])
2528 and _ris_multiline_line_carries_userinfo(
2529 entry_text, value_positions["doi"]
2530 )
2531 ):
2532 metadata["doi"] = ""
2534 # Combine title with additional metadata lines for full context
2535 full_text = entry_text
2537 # One record per index, keyed on the INDIVIDUAL index rather than
2538 # the group text: a report repeats its Sources block, and the same
2539 # source may be written ``[1]`` in one block and ``[1, 3]`` in
2540 # another, so keying on the raw group would let ``ref1`` be emitted
2541 # twice. Later definitions replace earlier ones, as they do in
2542 # the inline citation map. ``metadata`` is shared across the
2543 # indices of one line; ``_create_ris_entry`` only reads it.
2544 for citation_num in _iter_citation_group(citation_group):
2545 ris_entries[citation_num] = self._create_ris_entry(
2546 citation_num, full_text, url, metadata
2547 )
2549 return "\n".join(ris_entries.values())
2551 def _create_ris_entry(
2552 self,
2553 ref_id: str,
2554 full_text: str,
2555 url: str = "",
2556 metadata: dict | None = None,
2557 ) -> str:
2558 """Create a single RIS entry."""
2559 lines = []
2561 # Parse metadata from full text
2562 import re
2564 if metadata is None:
2565 metadata = {}
2567 # Extract title from first line. NB: split into a *separate* variable —
2568 # ``lines`` is the RIS-output accumulator initialized above and appended
2569 # to below; reusing it here previously overwrote it with the source
2570 # text, so every entry emitted the raw source body before the mandatory
2571 # leading ``TY - `` tag and reference managers rejected the file.
2572 text_lines = full_text.split("\n")
2573 title = text_lines[0].strip()
2575 # The year, the author list, and the DOI when no ``DOI:`` line gave
2576 # one, are cut out of whatever line their match captures from, and
2577 # the title is cleaned. A cut can drop a credential URL's @ and keep
2578 # its userinfo (the title ``https://DOI:pass@host`` yields the DOI
2579 # ``pass@host``), and a password can hold a year or a name
2580 # (``https://admin:2024@host`` yields the year 2024), so none of the
2581 # three is cut from a line that carries a credential
2582 # (``_ris_line_carries_userinfo``), and a flagged title is sanitized
2583 # whole, not cleaned. A title that its cleaning joins into a
2584 # credential URL gives no year or DOI either (``flagged_title``).
2585 year_match = re.search(r"\b(19\d{2}|20\d{2})\b", full_text)
2586 authors_match = re.search(
2587 r"\bby\s+([^.\n]+?)(?:\.|\n|$)", full_text, re.IGNORECASE
2588 )
2589 doi = metadata.get("doi")
2590 doi_match = (
2591 None
2592 if doi
2593 else re.search(r"DOI:\s*([^\s\n]+)", full_text, re.IGNORECASE)
2594 )
2595 # Where the author and DOI patterns begin to read a value; a line
2596 # is checked from each of them too. Each line is checked once.
2597 value_starts = tuple(
2598 match.start(1)
2599 for match in (authors_match, doi_match)
2600 if match is not None
2601 )
2602 line_carries_userinfo: dict[int, bool] = {}
2604 def cut_is_safe(match: re.Match) -> bool:
2605 line_start = full_text.rfind("\n", 0, match.start(1)) + 1
2606 if line_start not in line_carries_userinfo:
2607 line_carries_userinfo[line_start] = _ris_line_carries_userinfo(
2608 full_text, match.start(1), value_starts
2609 )
2610 return not line_carries_userinfo[line_start]
2612 # Extract year from full text (looks for 4-digit year)
2613 year = (
2614 year_match.group(1)
2615 if year_match and cut_is_safe(year_match)
2616 else None
2617 )
2619 # Extract authors if present (looks for "by Author1, Author2")
2620 authors = []
2621 if authors_match and cut_is_safe(authors_match):
2622 # The captured text and each author are checked as written too.
2623 authors_text = _ris_field_text(authors_match.group(1))
2624 # Split at delimiters only, then trim each author. Consuming
2625 # optional whitespace before a delimiter makes a long run with
2626 # no delimiter quadratic; lookarounds keep "and" word-scoped.
2627 author_parts = re.split(r",|(?<=\s)and(?=\s)|&", authors_text)
2628 authors = [
2629 author
2630 for author in (_ris_field_text(a.strip()) for a in author_parts)
2631 if author
2632 ]
2634 # Extract DOI from metadata or text
2635 if doi_match is not None:
2636 doi = doi_match.group(1) if cut_is_safe(doi_match) else None
2638 # Clean title - remove author and metadata info for cleaner title.
2639 # Every form a cut makes is kept for the credential check below.
2640 clean_title = title
2641 title_forms = [title]
2642 if authors_match and authors_match.start() < len(title):
2643 clean_title = (
2644 title[: authors_match.start()] + title[authors_match.end() :]
2645 if authors_match.end() < len(title)
2646 else title[: authors_match.start()]
2647 )
2648 title_forms.append(clean_title)
2649 for clause in _RIS_TITLE_CLAUSES:
2650 clean_title = _cut_title_clause(clause, clean_title)
2651 title_forms.append(clean_title)
2652 clean_title = clean_title.strip()
2653 title_forms.append(clean_title)
2655 # A title carries a credential when the check flags it as written or
2656 # in any form a cut makes of it: what a cut leaves can join into a
2657 # credential URL, and a later cut can cut that URL's @ away again,
2658 # leaving its userinfo. The first form the check flags is sanitized
2659 # whole, and the title gives no year and no DOI.
2660 flagged_title = next(
2661 (
2662 form
2663 for form in dict.fromkeys(title_forms)
2664 if _ris_spells_userinfo(form)
2665 ),
2666 None,
2667 )
2668 # Cleaning can remove a complete credential clause while leaving a
2669 # different fragment, or join a fragment to the continuation lines.
2670 # Check the title we would write together with the uncut remainder.
2671 fragmented_title = (
2672 flagged_title is None
2673 and _ris_multiline_line_carries_userinfo(
2674 (clean_title or title) + full_text[len(text_lines[0]) :], 0
2675 )
2676 )
2677 if flagged_title is not None or fragmented_title:
2678 title_end = len(text_lines[0])
2679 if year_match and year_match.start(1) < title_end:
2680 year = None
2681 if doi_match and doi_match.start(1) < title_end:
2682 doi = None
2684 # TY - Type of reference (ELEC for electronic source/website)
2685 lines.append("TY - ELEC")
2687 # ID - Reference ID
2688 lines.append(f"ID - ref{ref_id}")
2690 # TI - Title. A source's entire title can itself be a URL, or hold
2691 # one: a title the shared credential check flags is written as its
2692 # userinfo-free URL, or as a placeholder when ``_safe_ris_url``
2693 # keeps no URL for it. The title as written is checked first, since
2694 # cleaning can cut the @ out of a credential URL; each form a cut
2695 # makes is checked too, since joining what a cut leaves can spell
2696 # one (``flagged_title`` above).
2697 if fragmented_title:
2698 display_title = "[source URL omitted]"
2699 elif flagged_title is None:
2700 display_title = clean_title if clean_title else title
2701 else:
2702 display_title = (
2703 _safe_ris_url(flagged_title) or "[source URL omitted]"
2704 )
2705 lines.append(f"TI - {display_title}")
2707 # AU - Authors
2708 for author in authors:
2709 lines.append(f"AU - {author}")
2711 # DO - DOI. Often written as a doi.org URL, so it gets the same
2712 # credential check as the title; one it flags without a URL that
2713 # ``_safe_ris_url`` keeps is left out.
2714 if doi:
2715 doi = _ris_field_text(str(doi))
2716 if doi:
2717 lines.append(f"DO - {doi}")
2719 # PY - Publication year (if one was found above)
2720 if year:
2721 lines.append(f"PY - {year}")
2723 # UR - URL. Legacy reports and LLM-authored Sources may still carry
2724 # user:password@host even though new reports canonicalize their URLs.
2725 safe_url = _safe_ris_url(url)
2726 if safe_url:
2727 lines.append(f"UR - {safe_url}")
2729 # Try to extract domain as publisher
2730 try:
2731 from urllib.parse import urlparse
2733 parsed = urlparse(safe_url)
2734 # The host alone. The netloc also carries userinfo and port,
2735 # so https://user:token@arxiv.org/ wrote the token into the
2736 # .ris file as the publisher (#6837). The root dot of a fully
2737 # qualified name is dropped, as utilities/arxiv.py does.
2738 domain = (parsed.hostname or "").rstrip(".")
2739 if domain.startswith("www."):
2740 domain = domain[4:]
2741 # Extract readable publisher name from domain
2742 if domain == "github.com" or domain.endswith(".github.com"):
2743 lines.append("PB - GitHub")
2744 elif domain == "arxiv.org" or domain.endswith(".arxiv.org"):
2745 lines.append("PB - arXiv")
2746 elif domain == "reddit.com" or domain.endswith(".reddit.com"):
2747 lines.append("PB - Reddit")
2748 elif (
2749 domain == "youtube.com"
2750 or domain == "m.youtube.com"
2751 or domain.endswith(".youtube.com")
2752 ):
2753 lines.append("PB - YouTube")
2754 elif domain == "medium.com" or domain.endswith(".medium.com"):
2755 lines.append("PB - Medium")
2756 elif domain == "pypi.org" or domain.endswith(".pypi.org"):
2757 lines.append("PB - Python Package Index (PyPI)")
2758 else:
2759 # Use domain as publisher
2760 lines.append(f"PB - {domain}")
2761 except (ValueError, AttributeError):
2762 pass
2764 # Y1 - Year accessed (current year)
2765 from datetime import UTC, datetime
2767 current_year = datetime.now(UTC).year
2768 lines.append(f"Y1 - {current_year}")
2770 # DA - Date accessed
2771 current_date = datetime.now(UTC).strftime("%Y/%m/%d")
2772 lines.append(f"DA - {current_date}")
2774 # LA - Language
2775 lines.append("LA - en")
2777 # ER - End of reference
2778 lines.append("ER - ")
2780 # A field can contain bare CR or Unicode line separators even when
2781 # extracted from one LF-delimited source line. Flatten them at the
2782 # serialization boundary so only our separators create RIS fields.
2783 # The credential check reads every value flattened the same way.
2784 return "\n".join(_ris_flatten(line) for line in lines)
2787# Body prose is markdown, where "\$" is the escape for a literal
2788# dollar. The escaper therefore consumes a "\$" pair as ONE token and
2789# emits "\$" (a literal dollar in LaTeX too); every other special,
2790# every other backslash included, goes through the _escape_latex_text
2791# map. Pair-first alternation in a single regex pass means nothing is
2792# ever escaped twice: "\$" never becomes "\textbackslash{}\$" (which
2793# would print a stray backslash), and the output never holds "\\"
2794# (a line break) or a bare "$" (a live math shift).
2795_LATEX_BODY_TEXT_RE = re.compile(
2796 r"\\\$|[" + re.escape("".join(_LATEX_TEXT_SPECIALS)) + "]"
2797)
2800def _escape_latex_body_text(text: str) -> str:
2801 r"""Escape a prose segment of the export body in one pass.
2803 Identical to :func:`_escape_latex_text` except that a markdown
2804 ``\$`` (escaped dollar) becomes ``\$`` rather than
2805 ``\textbackslash{}\$``. A backslash run before a dollar pairs only
2806 its LAST backslash with the dollar; the others are literal
2807 backslashes (``\\$`` -> ``\textbackslash{}\$``).
2808 """
2809 return _LATEX_BODY_TEXT_RE.sub(
2810 lambda m: (
2811 "\\$" if m.group() == "\\$" else _LATEX_TEXT_SPECIALS[m.group()]
2812 ),
2813 text,
2814 )
2817# A blank line ends a block: two line breaks separated only by
2818# horizontal whitespace and carriage returns. Legacy report_content
2819# rows store CRLF newlines and LLM prose often leaves spaces or tabs
2820# on the otherwise blank line, so probing for a literal "\n\n" misses
2821# real paragraph boundaries; the boundary is matched structurally.
2822_BLOCK_END = re.compile(r"\r?\n[ \t\r]*\r?\n")
2824# Verdicts _split_math_segments assigns to each live "$" as the closer
2825# of the opener before it: acceptable; rejected because a digit follows
2826# it (a price: "$5-$10"); or rejected because whitespace precedes it.
2827_CLOSER_OK = 0
2828_CLOSER_DIGIT = 1
2829_CLOSER_SPACE = 2
2832def _split_math_segments(text: str) -> list[tuple[str, bool]]:
2833 r"""Split *text* into ``(segment, is_math)`` pairs, left to right.
2835 ``$$...$$`` display spans pair first, then ``$...$`` inline spans.
2836 Neither kind ever crosses a blank line (pandoc scopes math to a
2837 single block, so a stray ``$$`` or a price in one paragraph can
2838 never pair with a delimiter in another; an opener unpaired within
2839 its block is literal text). An inline span's opening ``$`` must
2840 not be followed by whitespace, and it closes at the FIRST live
2841 ``$`` after it in the block, as in pandoc: that closer must not be
2842 preceded by whitespace and must not be followed by a digit, and if
2843 it is, the OPENER fails and reads as prose (``$5-$10`` is two
2844 prices, not ``$5-$`` around ``10``). Rejected closers are never
2845 stepped over on the way to a farther one, which would emit a span
2846 holding a raw interior dollar. A closer rejected for the digit
2847 after it is itself a price and cannot open a span either; pandoc
2848 would let it open one, so ``$5-$10 per unit ($n$)`` would make
2849 ``$10 per unit ($`` math there, raw prose between two prices
2850 (whitespace-rejected closers still open: ``$5 so $x$`` keeps
2851 ``$x$``). Mistaking math for prose only escapes it; mistaking
2852 prose for math emits its ``%`` and ``&`` raw. (Mirror case: because
2853 the rejected closer cannot open, a later ``$`` glued to the text on
2854 its left can open instead and pair farther on, so
2855 ``$5-$10 per hour (US$), 50% more (US$)`` makes ``$), 50% more (US$``
2856 math; this is rarer than the cases the rule prevents.) A ``$``
2857 preceded by an odd run of backslashes is a literal dollar
2858 (as in ``\$5``), never an opener or a closer; after an even run
2859 (``\\$``: the backslashes escape each other) the dollar is as
2860 live as a bare one, in both directions. Every other dollar is a
2861 literal character inside a text segment. Math segments include their
2862 delimiters and are returned verbatim.
2863 """
2864 segments: list[tuple[str, bool]] = []
2865 start = 0 # start of the current text run
2866 i = 0
2867 n = len(text)
2868 # Whether a "$" can close inline math does not depend on the
2869 # opener, so every live candidate position and its verdict is
2870 # collected once up front; the pairing below advances one pointer
2871 # over them, left to right, and never rescans. A "$" ending an odd
2872 # backslash run is a literal dollar and never a delimiter in
2873 # either direction: excluded here so the walk cannot CLOSE a span
2874 # on it, exactly as the "\x" skip below already keeps it from
2875 # opening one. A "$" after an even run ("\\$": the backslashes
2876 # escape each other) is live and stays a closer candidate; a
2877 # single-character lookbehind excluded it too and left "$x\\$"
2878 # unpaired. That pattern's lookbehind stops it starting mid-run,
2879 # so each match spans the full run and the scan stays linear
2880 # even on backslash walls.
2881 odd_run_dollars = {
2882 m.end() - 1 for m in re.finditer(r"(?<!\\)(?:\\\\)*\\\$", text)
2883 }
2884 dollars = [
2885 m.start()
2886 for m in re.finditer(r"\$", text)
2887 if m.start() not in odd_run_dollars
2888 ]
2889 # Whitespace is checked first: a candidate with both flaws ("$fee
2890 # $5$") is not a price and may still open its own span.
2891 closer_verdict = [
2892 _CLOSER_SPACE
2893 if p > 0 and text[p - 1].isspace()
2894 else _CLOSER_DIGIT
2895 if p + 1 < n and text[p + 1] in "0123456789"
2896 else _CLOSER_OK
2897 for p in dollars
2898 ]
2899 cand = 0 # index into dollars: first candidate after the opener
2900 price_dollar = -1 # a digit-rejected closer: never an opener
2901 # First blank line at or after the current position.
2902 block_end = n
2903 block_end_known = False
2904 while i < n:
2905 if text[i] == "\\":
2906 # A backslash escapes the next character: "\$" is a
2907 # literal dollar, never a math delimiter.
2908 i += 2
2909 continue
2910 if text[i] != "$":
2911 i += 1
2912 continue
2913 if i + 1 < n and text[i + 1] == "$":
2914 # A display opener pairs only inside its own block, so a
2915 # stray "$$" in one paragraph cannot capture a genuine
2916 # display span in another (a blank line is never inside
2917 # math). The closer scan walks the block: "\x" pairs are
2918 # skipped (an interior "\$" is a literal dollar, and the
2919 # "$" of "\$$" is not the lead of a closer), the first
2920 # raw single "$" means the opener is unpaired (a display
2921 # span has no single-dollar interior), and the first raw
2922 # "$$" closes.
2923 if not block_end_known or i > block_end:
2924 boundary = _BLOCK_END.search(text, i)
2925 block_end = boundary.start() if boundary is not None else n
2926 block_end_known = True
2927 close = -1
2928 j = i + 2
2929 while j < block_end:
2930 if text[j] == "\\":
2931 j += 2
2932 elif text[j] == "$":
2933 if j + 1 < n and text[j + 1] == "$":
2934 close = j
2935 break
2936 else:
2937 j += 1
2938 if close != -1:
2939 segments.append((text[start:i], False))
2940 segments.append((text[i : close + 2], True))
2941 i = close + 2
2942 start = i
2943 else:
2944 # Unpaired $$ within its block: literal text, like
2945 # any lone dollar.
2946 i += 2
2947 continue
2948 # Single '$': find a closing '$' the rules accept, without
2949 # leaving the paragraph (inline math never crosses a blank
2950 # line).
2951 if i == price_dollar or (i + 1 < n and text[i + 1].isspace()):
2952 i += 1
2953 continue
2954 # The first blank line at or after i only moves forward as i
2955 # does, so the search is cached until i passes its match.
2956 if not block_end_known or i > block_end:
2957 boundary = _BLOCK_END.search(text, i)
2958 block_end = boundary.start() if boundary is not None else n
2959 block_end_known = True
2960 # The closer is the first live "$" after the opener. Openers
2961 # only move right, so the pointer does too.
2962 while cand < len(dollars) and dollars[cand] <= i:
2963 cand += 1
2964 verdict = (
2965 closer_verdict[cand]
2966 if cand < len(dollars) and dollars[cand] < block_end
2967 else None
2968 )
2969 if verdict == _CLOSER_OK:
2970 closer = dollars[cand]
2971 segments.append((text[start:i], False))
2972 segments.append((text[i : closer + 1], True))
2973 i = closer + 1
2974 start = i
2975 else:
2976 # No closer in this block, or the first one is preceded
2977 # by whitespace or followed by a digit: the opener fails
2978 # (never stepped past to a farther "$") and is a literal
2979 # dollar. A digit-rejected closer is a price as well.
2980 if verdict == _CLOSER_DIGIT:
2981 price_dollar = dollars[cand]
2982 i += 1
2983 segments.append((text[start:], False))
2984 return segments
2987# Math in a report may come from retrieved pages through the LLM. TeX executes
2988# control sequences when a user compiles an exported .tex file, so preserving
2989# every matched $...$ span verbatim is unsafe. Only math commands that the
2990# LaTeX kernel itself defines are accepted, because the export preamble loads
2991# no math package: \text, \binom and \operatorname (amsmath) and \mathbb
2992# (amsfonts) would be undefined there, so spans using them are rendered as
2993# literal text. Each accepted command is a math symbol, delimiter
2994# or accent, or a TeX primitive or kernel macro that only typesets (boxes,
2995# spacing, font changes). None of them lets the report's text open or write
2996# a file, run a shell command, change a category code or define a macro.
2997# Every character in an argument is checked by the same scanner, so an
2998# allowed command cannot hide a second one.
2999_SAFE_MATH_COMMANDS = frozenset(
3000 {
3001 # Greek letters, lower case. There is no \omicron: it is "o".
3002 "alpha",
3003 "beta",
3004 "gamma",
3005 "delta",
3006 "epsilon",
3007 "varepsilon",
3008 "zeta",
3009 "eta",
3010 "theta",
3011 "vartheta",
3012 "iota",
3013 "kappa",
3014 "lambda",
3015 "mu",
3016 "nu",
3017 "xi",
3018 "pi",
3019 "varpi",
3020 "rho",
3021 "varrho",
3022 "sigma",
3023 "varsigma",
3024 "tau",
3025 "upsilon",
3026 "phi",
3027 "varphi",
3028 "chi",
3029 "psi",
3030 "omega",
3031 # Greek letters, upper case. The others (A, B, E, ...) are Latin.
3032 "Gamma",
3033 "Delta",
3034 "Theta",
3035 "Lambda",
3036 "Xi",
3037 "Pi",
3038 "Sigma",
3039 "Upsilon",
3040 "Phi",
3041 "Psi",
3042 "Omega",
3043 # Binary operators.
3044 "pm",
3045 "mp",
3046 "times",
3047 "div",
3048 "cdot",
3049 "ast",
3050 "star",
3051 "circ",
3052 "bullet",
3053 "setminus",
3054 "cap",
3055 "cup",
3056 "uplus",
3057 "sqcap",
3058 "sqcup",
3059 "vee",
3060 "wedge",
3061 "land",
3062 "lor",
3063 "oplus",
3064 "ominus",
3065 "otimes",
3066 "oslash",
3067 "odot",
3068 "dagger",
3069 "ddagger",
3070 "amalg",
3071 "diamond",
3072 "wr",
3073 "bigcirc",
3074 "triangleleft",
3075 "triangleright",
3076 "bigtriangleup",
3077 "bigtriangledown",
3078 # Relations, including the \le, \ge and \ne short forms and \not,
3079 # which overlays a slash on the relation after it.
3080 "leq",
3081 "le",
3082 "geq",
3083 "ge",
3084 "neq",
3085 "ne",
3086 "ll",
3087 "gg",
3088 "prec",
3089 "succ",
3090 "preceq",
3091 "succeq",
3092 "sim",
3093 "simeq",
3094 "approx",
3095 "cong",
3096 "equiv",
3097 "doteq",
3098 "propto",
3099 "asymp",
3100 "perp",
3101 "parallel",
3102 "mid",
3103 "models",
3104 "vdash",
3105 "dashv",
3106 "smile",
3107 "frown",
3108 "bowtie",
3109 "in",
3110 "ni",
3111 "owns",
3112 "notin",
3113 "subset",
3114 "supset",
3115 "subseteq",
3116 "supseteq",
3117 "sqsubseteq",
3118 "sqsupseteq",
3119 "not",
3120 # Arrows.
3121 "to",
3122 "gets",
3123 "leftarrow",
3124 "rightarrow",
3125 "leftrightarrow",
3126 "Leftarrow",
3127 "Rightarrow",
3128 "Leftrightarrow",
3129 "longleftarrow",
3130 "longrightarrow",
3131 "longleftrightarrow",
3132 "Longleftarrow",
3133 "Longrightarrow",
3134 "Longleftrightarrow",
3135 "iff",
3136 "mapsto",
3137 "longmapsto",
3138 "hookleftarrow",
3139 "hookrightarrow",
3140 "uparrow",
3141 "downarrow",
3142 "updownarrow",
3143 "Uparrow",
3144 "Downarrow",
3145 "Updownarrow",
3146 "nearrow",
3147 "searrow",
3148 "swarrow",
3149 "nwarrow",
3150 "leftharpoonup",
3151 "leftharpoondown",
3152 "rightharpoonup",
3153 "rightharpoondown",
3154 "rightleftharpoons",
3155 # Other symbols.
3156 "infty",
3157 "partial",
3158 "nabla",
3159 "forall",
3160 "exists",
3161 "neg",
3162 "lnot",
3163 "emptyset",
3164 "aleph",
3165 "hbar",
3166 "imath",
3167 "jmath",
3168 "ell",
3169 "wp",
3170 "Re",
3171 "Im",
3172 "prime",
3173 "top",
3174 "bot",
3175 "angle",
3176 "triangle",
3177 "surd",
3178 "flat",
3179 "natural",
3180 "sharp",
3181 "clubsuit",
3182 "diamondsuit",
3183 "heartsuit",
3184 "spadesuit",
3185 "colon",
3186 "ldots",
3187 "cdots",
3188 "vdots",
3189 "ddots",
3190 "dots",
3191 # Big operators.
3192 "sum",
3193 "prod",
3194 "coprod",
3195 "int",
3196 "oint",
3197 "smallint",
3198 "bigcap",
3199 "bigcup",
3200 "bigsqcup",
3201 "bigvee",
3202 "bigwedge",
3203 "bigodot",
3204 "bigoplus",
3205 "bigotimes",
3206 "biguplus",
3207 # Named operators, set upright like "log".
3208 "arccos",
3209 "arcsin",
3210 "arctan",
3211 "arg",
3212 "cos",
3213 "cosh",
3214 "cot",
3215 "coth",
3216 "csc",
3217 "deg",
3218 "det",
3219 "dim",
3220 "exp",
3221 "gcd",
3222 "hom",
3223 "inf",
3224 "ker",
3225 "lg",
3226 "lim",
3227 "liminf",
3228 "limsup",
3229 "ln",
3230 "log",
3231 "max",
3232 "min",
3233 "Pr",
3234 "sec",
3235 "sin",
3236 "sinh",
3237 "sup",
3238 "tan",
3239 "tanh",
3240 "bmod",
3241 "pmod",
3242 # Delimiters and the commands that size them (\middle is e-TeX).
3243 "left",
3244 "right",
3245 "middle",
3246 "big",
3247 "Big",
3248 "bigg",
3249 "Bigg",
3250 "bigl",
3251 "bigr",
3252 "Bigl",
3253 "Bigr",
3254 "biggl",
3255 "biggr",
3256 "Biggl",
3257 "Biggr",
3258 "bigm",
3259 "Bigm",
3260 "biggm",
3261 "Biggm",
3262 "langle",
3263 "rangle",
3264 "lfloor",
3265 "rfloor",
3266 "lceil",
3267 "rceil",
3268 "lbrace",
3269 "rbrace",
3270 "lbrack",
3271 "rbrack",
3272 "vert",
3273 "Vert",
3274 "backslash",
3275 "lgroup",
3276 "rgroup",
3277 "lmoustache",
3278 "rmoustache",
3279 "arrowvert",
3280 "Arrowvert",
3281 "bracevert",
3282 # Accents, and lines, braces and arrows set over or under a formula.
3283 "hat",
3284 "check",
3285 "tilde",
3286 "acute",
3287 "grave",
3288 "dot",
3289 "ddot",
3290 "breve",
3291 "bar",
3292 "vec",
3293 "mathring",
3294 "widehat",
3295 "widetilde",
3296 "overline",
3297 "underline",
3298 "overbrace",
3299 "underbrace",
3300 "overrightarrow",
3301 "overleftarrow",
3302 # Fractions and roots.
3303 "frac",
3304 "sqrt",
3305 # Math alphabets.
3306 "mathrm",
3307 "mathbf",
3308 "mathit",
3309 "mathsf",
3310 "mathtt",
3311 "mathcal",
3312 "mathnormal",
3313 # Math styles and the placement of limits.
3314 "displaystyle",
3315 "textstyle",
3316 "scriptstyle",
3317 "scriptscriptstyle",
3318 "limits",
3319 "nolimits",
3320 # Spacing. \medspace and \thickspace are left out: the kernel only
3321 # gained them in 2020, so older TeX installations lack them.
3322 "quad",
3323 "qquad",
3324 "thinspace",
3325 "negthinspace",
3326 "enspace",
3327 }
3328)
3329# Control symbols: a backslash and one non-letter. \$ and \\ are the
3330# literal-dollar and escaped-backslash spans the delimiter scanner already
3331# supports. \{, \} and \| are delimiters; \, \: \; \! and "\ " are spaces;
3332# \% \# \& and \_ print the character. "\^" stays out: it would let "\^^"
3333# hide a ^^ encoding from the check in _math_span_is_safe.
3334_SAFE_MATH_CONTROL_SYMBOLS = frozenset("$\\{}|,:;! %#&_")
3335_ASCII_MATH_LETTERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
3336# "'" is the prime, as in f'(x). "\r" is handled in _math_span_is_safe.
3337_SAFE_MATH_CHARACTERS = frozenset(
3338 _ASCII_MATH_LETTERS + "0123456789 \t\n+-*/=.,:;!?(){}[]^_<>|'"
3339)
3342def _math_span_is_safe(segment: str) -> bool:
3343 """Accept only inert math syntax inside a matched inline/display span."""
3344 delimiter = "$$" if segment.startswith("$$") else "$"
3345 body = segment[len(delimiter) : -len(delimiter)]
3346 depth = 0
3347 position = 0
3348 while position < len(body):
3349 character = body[position]
3350 if character == "\\":
3351 position += 1
3352 if position == len(body): 3352 ↛ 3353line 3352 didn't jump to line 3353 because the condition on line 3352 was never true
3353 return False
3354 if body[position] in _ASCII_MATH_LETTERS:
3355 start = position
3356 while (
3357 position < len(body)
3358 and body[position] in _ASCII_MATH_LETTERS
3359 ):
3360 position += 1
3361 if body[start:position] not in _SAFE_MATH_COMMANDS:
3362 return False
3363 continue
3364 if body[position] not in _SAFE_MATH_CONTROL_SYMBOLS:
3365 return False
3366 position += 1
3367 continue
3368 if character == "\r":
3369 # Legacy report_content rows store CRLF newlines, and TeX reads a
3370 # CRLF pair as one line break. TeX also ends a line at a lone CR,
3371 # which _BLOCK_END does not see, so "\r\r" could hide a blank line
3372 # (a paragraph break) inside a span; a lone CR stays literal.
3373 if not body.startswith("\n", position + 1):
3374 return False
3375 position += 1
3376 continue
3377 if character not in _SAFE_MATH_CHARACTERS:
3378 return False
3379 # TeX turns ^^5c (and longer ^^ encodings) into a backslash before
3380 # it recognizes control sequences. A character-only allowlist would
3381 # otherwise accept an encoded \input with no literal backslash.
3382 if character == "^" and body.startswith("^^", position):
3383 return False
3384 if character == "{":
3385 depth += 1
3386 elif character == "}":
3387 depth -= 1
3388 if depth < 0: 3388 ↛ 3389line 3388 didn't jump to line 3389 because the condition on line 3388 was never true
3389 return False
3390 position += 1
3391 return depth == 0
3394# Pandoc's markdown reader treats a backslash before any ASCII punctuation
3395# character as that literal character (the ``all_symbols_escapable``
3396# extension, on by default), so escaping every punctuation character in a
3397# span renders it as text: a ``$`` no longer shifts into math
3398# (``tex_math_dollars`` needs an unescaped ``$``) and the backslash of a
3399# command such as ``\input`` becomes a literal backslash, not a control
3400# sequence, even while ``raw_tex`` is on. Letters and digits are left as
3401# they are; only the characters that could carry meaning are neutralised.
3402_MARKDOWN_PUNCTUATION_RE = re.compile(r"[!-/:-@\[-`{-~]")
3405def _escape_markdown_math_span(segment: str) -> str:
3406 r"""Render a rejected ``$...$``/``$$...$$`` span as inert Markdown text.
3408 Backslash-escapes every ASCII punctuation character, so Pandoc reads the
3409 span as literal characters rather than math or raw TeX.
3410 """
3411 return _MARKDOWN_PUNCTUATION_RE.sub(r"\\\g<0>", segment)
3414def _defuse_report_math_for_quarto(content: str) -> str:
3415 r"""Escape every unsafe math span in a Quarto report body.
3417 The Quarto front matter declares a ``pdf`` format, so ``quarto render``
3418 hands each ``$...$``/``$$...$$`` span to Pandoc's LaTeX writer, which
3419 emits math verbatim. A span is kept as math only when
3420 :func:`_math_span_is_safe` accepts it — the same allowlist the LaTeX
3421 export applies — and every other span is escaped to literal text so it
3422 never reaches the TeX engine. Prose between spans is returned untouched
3423 for the citation and cell passes that run after it.
3424 """
3425 parts: list[str] = []
3426 for segment, is_math in _split_math_segments(content):
3427 if is_math and not _math_span_is_safe(segment):
3428 parts.append(_escape_markdown_math_span(segment))
3429 else:
3430 parts.append(segment)
3431 return "".join(parts)
3434# The start of a citation link as CitationFormatter writes it: the label
3435# in double brackets, then the destination. Every hyperlinked mode uses this
3436# shape; only the label differs ([[1]], [[arxiv.org]], [[arxiv.org-2]],
3437# [[doi-1]]).
3438_CITATION_LINK_START_RE = re.compile(r"\[\[([^\[\]\n]+)\]\]\(")
3439_CITATION_GROUP_LABEL_RE = re.compile(r"\s*\d+(?:\s*,\s*\d+)+\s*")
3442def _link_destination_ends(text: str) -> array:
3443 """For every index, where a link destination starting there ends.
3445 ``_safe_link_destination`` percent-encodes spaces and any unbalanced
3446 parenthesis, so a destination it wrote ends at the first ``)`` that
3447 closes no ``(`` of its own, as in ``https://en.wikipedia.org/wiki/A_(b)``.
3448 ``ends[i]`` is the index of that ``)`` for a destination starting at
3449 ``i``, or -1 when whitespace or the end of the text comes first.
3451 One pass from the right answers every start at once. Scanning forward
3452 from each ``[[a]](`` instead costs quadratic time on a text full of
3453 destinations that never close: 20 000 of them took about 40 seconds.
3454 """
3455 ends = array("q", [-1]) * (len(text) + 1)
3456 for position in range(len(text) - 1, -1, -1):
3457 character = text[position]
3458 if character == ")":
3459 ends[position] = position
3460 elif character == "(":
3461 # This parenthesis is closed where a destination starting just
3462 # after it would end; the destination then carries on from there.
3463 inner = ends[position + 1]
3464 ends[position] = -1 if inner == -1 else ends[inner + 1]
3465 elif not character.isspace():
3466 ends[position] = ends[position + 1]
3467 return ends
3470class LaTeXExporter:
3471 """Export markdown documents to LaTeX format."""
3473 def __init__(self):
3474 # Also match Unicode lenticular brackets 【】 (U+3010 and U+3011) that LLMs sometimes generate
3475 self.citation_pattern = re.compile(r"[\[【](\d+)[\]】]")
3476 self.comma_citation_pattern = re.compile(
3477 r"[\[【](\d+(?:\s*,\s*\d+)+)[\]】]" # (#6944) space allowed BEFORE the comma too — a hand-edited `[[2 ,3]]` must stay a group
3478 )
3479 self.heading_patterns = [
3480 (re.compile(r"^# (.+)$", re.MULTILINE), r"\\section{\1}"),
3481 (re.compile(r"^## (.+)$", re.MULTILINE), r"\\subsection{\1}"),
3482 (re.compile(r"^### (.+)$", re.MULTILINE), r"\\subsubsection{\1}"),
3483 ]
3484 self.emphasis_patterns = [
3485 (re.compile(r"\*\*(.+?)\*\*"), r"\\textbf{\1}"),
3486 (re.compile(r"\*(.+?)\*"), r"\\textit{\1}"),
3487 (re.compile(r"`(.+?)`"), r"\\texttt{\1}"),
3488 ]
3490 def export_to_latex(self, content: str) -> str:
3491 """
3492 Convert markdown document to LaTeX format.
3494 Args:
3495 content: Markdown content
3497 Returns:
3498 LaTeX formatted content
3499 """
3500 latex_content = self._create_latex_header()
3502 # Convert markdown to LaTeX. A saved report was citation-formatted
3503 # before it was stored, so its citations arrive as links.
3504 entries = self._sources_entries(content)
3505 body_content = self._unlink_citations(content, entries)
3507 # Escape special LaTeX characters while preserving safe math. A
3508 # left-to-right scanner pairs $$...$$ display spans and $...$
3509 # inline spans; accepted spans pass through verbatim and rejected
3510 # spans are rendered as literal text. Every other dollar
3511 # is prose. The previous whole-document split made ONE unpaired
3512 # dollar (a price in prose) invert the parity and silently
3513 # disable escaping for the rest of the report.
3514 escaped_parts: list[str] = []
3515 at_line_start = True
3516 for segment, is_math in _split_math_segments(body_content):
3517 if is_math and _math_span_is_safe(segment):
3518 escaped_parts.append(segment)
3519 else:
3520 lines = segment.split("\n")
3521 for j, line in enumerate(lines):
3522 # Escape every LaTeX-special character outside math
3523 # mode. Markdown markers (*, `, [n]) are not
3524 # LaTeX-special and survive for their own
3525 # conversions below.
3526 heading = (
3527 _HEADING_PREFIX_RE.match(line)
3528 if j > 0 or at_line_start
3529 else None
3530 )
3531 if heading:
3532 # Keep the '#' markers intact for the heading
3533 # conversion; escape the text after them so
3534 # \section{...} receives literal text only. Only
3535 # a TRUE line start qualifies: a text segment
3536 # that resumes mid-line after a math span
3537 # ("$x$ # y") is prose, and its "#" is escaped.
3538 lines[j] = heading.group(1) + _escape_latex_body_text(
3539 heading.group(2)
3540 )
3541 else:
3542 lines[j] = _escape_latex_body_text(line)
3543 escaped_parts.append("\n".join(lines))
3544 if segment:
3545 at_line_start = segment.endswith("\n")
3546 body_content = "".join(escaped_parts)
3548 # Convert headings
3549 for pattern, replacement in self.heading_patterns:
3550 body_content = pattern.sub(replacement, body_content)
3552 # Convert emphasis
3553 for pattern, replacement in self.emphasis_patterns:
3554 body_content = pattern.sub(replacement, body_content)
3556 # Convert citations to LaTeX \cite{} format
3557 # A group becomes one \\cite only when the Sources section lists every
3558 # member, as CitationFormatter only links the members it can resolve;
3559 # otherwise it is prose such as "versions [4, 5]" and is left as written.
3560 def cite_group(match: re.Match[str]) -> str:
3561 nums = [num.strip() for num in match.group(1).split(",")]
3562 if all(num in entries for num in nums):
3563 return "\\cite{" + ",".join(nums) + "}"
3564 return match.group(0)
3566 body_content = self.comma_citation_pattern.sub(cite_group, body_content)
3567 body_content = self.citation_pattern.sub(r"\\cite{\1}", body_content)
3569 # Convert lists
3570 body_content = self._convert_lists(body_content)
3572 # Add body content
3573 latex_content += body_content
3575 # Add bibliography section
3576 latex_content += self._create_bibliography(content)
3578 # Add footer
3579 latex_content += self._create_latex_footer()
3581 return latex_content
3583 def _sources_entries(self, content: str) -> dict:
3584 """The Sources entries, from the same slice the bibliography reads."""
3585 sources_start, _on_sentinel = find_sources_section(content)
3586 if sources_start == -1:
3587 return {}
3588 return _collect_bibliography_entries(content[sources_start:])
3590 def _unlink_citations(self, content: str, entries: dict) -> str:
3591 """Turn the formatter's ``[[label]](url)`` citations back into ``[N]``.
3593 A number label is the citation index. A group label such as ``1, 2``
3594 names every member, which its one URL cannot, so it becomes ``[1, 2]``
3595 when the Sources list has every member. Any other label
3596 (``arxiv.org``, ``arxiv.org-2``, ``doi-1``) says nothing reliable
3597 about the index, so the link's destination is looked up among the
3598 Sources entries instead; when two entries share a URL, the lower
3599 index is used, since both name the same document. A link that
3600 resolves to no entry is left as written.
3601 """
3602 # Keyed on the destination the formatter would have written. Two URLs
3603 # that differ only in how a character is spelled, such as `a(b` and
3604 # `a%28b`, share a key, and the lower index keeps it.
3605 index_by_destination: Dict[str, str] = {}
3606 for index in sorted(entries, key=_sort_key):
3607 destination = _safe_link_destination(entries[index][1])
3608 if destination:
3609 index_by_destination.setdefault(destination, index)
3611 parts = []
3612 position = 0
3613 ends = None
3614 for match in _CITATION_LINK_START_RE.finditer(content):
3615 if match.start() < position:
3616 continue
3617 if ends is None:
3618 ends = _link_destination_ends(content)
3619 end = ends[match.end()]
3620 if end == -1:
3621 continue
3622 label = match.group(1)
3623 if label.isdecimal():
3624 index = label
3625 elif _CITATION_GROUP_LABEL_RE.fullmatch(label):
3626 members = [member.strip() for member in label.split(",")]
3627 if not all(member in entries for member in members):
3628 continue
3629 index = label.strip()
3630 else:
3631 index = index_by_destination.get(content[match.end() : end])
3632 if index is None:
3633 continue
3634 parts.append(content[position : match.start()])
3635 parts.append(f"[{index}]")
3636 position = end + 1
3637 parts.append(content[position:])
3638 return "".join(parts)
3640 def _create_latex_header(self) -> str:
3641 """Create LaTeX document header."""
3642 return r"""\documentclass[12pt]{article}
3643\usepackage[utf8]{inputenc}
3644\usepackage{hyperref}
3645\usepackage{cite}
3646\usepackage{url}
3648\title{Research Report}
3649\date{\today}
3651\begin{document}
3652\maketitle
3654"""
3656 def _create_latex_footer(self) -> str:
3657 """Create LaTeX document footer."""
3658 return "\n\\end{document}\n"
3660 def _escape_latex(self, text: str) -> str:
3661 """Escape special LaTeX characters in text."""
3662 return _escape_latex_text(text)
3664 def _convert_lists(self, content: str) -> str:
3665 """Convert markdown lists to LaTeX format."""
3666 # Simple conversion for bullet points
3667 content = re.sub(r"^- (.+)$", r"\\item \1", content, flags=re.MULTILINE)
3669 # Add itemize environment around list items
3670 lines = content.split("\n")
3671 result = []
3672 in_list = False
3674 for line in lines:
3675 if line.strip().startswith("\\item"):
3676 if not in_list:
3677 result.append("\\begin{itemize}")
3678 in_list = True
3679 result.append(line)
3680 else:
3681 if in_list and line.strip():
3682 result.append("\\end{itemize}")
3683 in_list = False
3684 result.append(line)
3686 if in_list:
3687 result.append("\\end{itemize}")
3689 return "\n".join(result)
3691 def _create_bibliography(self, content: str) -> str:
3692 """Extract sources and create LaTeX bibliography."""
3693 sources_start, _on_sentinel = find_sources_section(content)
3694 if sources_start == -1:
3695 return ""
3697 sources_content = content[sources_start:]
3698 entries = _collect_bibliography_entries(sources_content)
3700 # The argument is the widest label, which sets the hanging indent. It
3701 # is never narrower than the two digits it has always been.
3702 widest = "9" * max([2, *(len(key) for key in entries)])
3703 bibliography = f"\n\\begin{{thebibliography}}{{{widest}}}\n"
3705 for citation_num in sorted(entries, key=_sort_key):
3706 title, url = entries[citation_num]
3707 escaped_title = self._escape_latex(title)
3709 # thebibliography numbers entries by POSITION unless a label
3710 # is given, so without one a gap in the index sequence makes
3711 # \\cite{5} print [3]. The label is the index the report shows,
3712 # as written: 007 stays 007, and a non-digit key such as a 7a
3713 # sub-index would be printed as it is too.
3714 item = f"\\bibitem[{citation_num}]{{{citation_num}}}"
3715 safe_url = _safe_bibtex_url(url)
3716 if safe_url:
3717 bibliography += f"{item} {escaped_title}. \\url{{{safe_url}}}\n"
3718 else:
3719 bibliography += f"{item} {escaped_title}.\n"
3721 bibliography += "\\end{thebibliography}\n"
3723 return bibliography