Coverage for src/local_deep_research/text_optimization/citation_formatter.py: 97%
545 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""Citation formatter for adding hyperlinks and alternative citation styles."""
3import re
4from enum import Enum
5from typing import Any, Dict, List, Optional, Tuple
6from urllib.parse import urlparse
8from loguru import logger
9from slugify import slugify
11from ..content_fetcher.url_classifier import URLClassifier, URLType
14# Marker emitted by ``IntegratedReportGenerator._format_final_report``
15# around the ``## Sources`` block it appends to the end of a detailed-mode
16# report. Looked up by ``find_sources_section`` so the splitter can locate
17# the appended sources section unambiguously, even when the LLM has
18# emitted its own ``## Sources`` (or ``References:`` etc.) header earlier
19# in the prose. The sentinel is a markdown HTML comment so it round-trips
20# through markdown renderers without affecting the displayed output.
21LDR_APPENDED_SOURCES_SENTINEL = "<!-- LDR_APPENDED_SOURCES_START -->"
23_SOURCES_SECTION_PATTERNS = [
24 re.compile(
25 r"^#{1,3}\s*(?:Sources|References|Bibliography|Citations)",
26 re.MULTILINE | re.IGNORECASE,
27 ),
28 re.compile(
29 r"^(?:Sources|References|Bibliography|Citations):?\s*$",
30 re.MULTILINE | re.IGNORECASE,
31 ),
32]
35def find_sources_section(
36 content: str, *, trust_sentinel: bool = True
37) -> Tuple[int, bool]:
38 """Locate the sources/references section in *content*.
40 Returns ``(position, on_sentinel)``. ``position`` is the character
41 offset of the section start, or ``-1`` if no section is found.
42 ``on_sentinel`` is ``True`` iff the unique sentinel emitted by the
43 report generator was matched (in which case the boundary is correct
44 by construction); ``False`` if the position came from the legacy
45 regex patterns (which can match an early ``## Sources`` line inside
46 the answer body and need an over-strip safety check downstream).
48 The sentinel is matched at its **last** occurrence (``rfind``) so a
49 spoofed earlier occurrence inside LLM-generated prose — e.g. the
50 LLM quoting the marker string verbatim while summarizing the
51 codebase — cannot silently truncate the answer body or disable the
52 over-strip safety check. Only a sentinel on its own line is trusted;
53 inline occurrences are skipped while searching backwards. The report
54 generator only ever emits the sentinel once at the end of the assembled
55 report, so the last standalone occurrence is the trusted one.
57 When ``trust_sentinel`` is ``False`` (e.g. raw LLM output from the
58 quick-mode save path, which never adds the sentinel itself), the
59 sentinel branch is skipped — only the legacy regex patterns run, so
60 a spurious ``on_sentinel=True`` cannot arise from the LLM quoting
61 the marker.
62 """
63 if trust_sentinel:
64 sentinel_pos = content.rfind(LDR_APPENDED_SOURCES_SENTINEL)
65 while sentinel_pos != -1:
66 preceded = sentinel_pos == 0 or content[sentinel_pos - 1] in (
67 "\n",
68 "\r",
69 )
70 after = content[sentinel_pos + len(LDR_APPENDED_SOURCES_SENTINEL) :]
71 followed = not after or after[0] in ("\n", "\r")
72 if preceded and followed:
73 return sentinel_pos, True
74 sentinel_pos = content.rfind(
75 LDR_APPENDED_SOURCES_SENTINEL, 0, sentinel_pos
76 )
77 for pattern in _SOURCES_SECTION_PATTERNS:
78 match = pattern.search(content)
79 if match:
80 return match.start(), False
81 return -1, False
84_BIB_SOURCES_PATTERN = re.compile(
85 r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
86 re.MULTILINE,
87)
90# Single-pass character map. A sequential replace() chain would re-escape
91# the braces its own replacements introduce — the bug already present in
92# _escape_latex, where "\\" becomes "\textbackslash{}" and the later
93# brace rules then mangle it.
94_BIBTEX_ESCAPES = {
95 "\\": r"\textbackslash{}",
96 # Balanced pairs, not backslash escapes: BibTeX's field scanner counts
97 # brace CHARACTERS and has no escape mechanism, so "\{" is still an open
98 # brace to it. A backslashed brace left the value unbalanced, and the
99 # scanner ran to EOF swallowing the rest of the .bib.
100 "{": r"{\textbraceleft}",
101 "}": r"{\textbraceright}",
102 "&": r"\&",
103 "%": r"\%",
104 "$": r"\$",
105 "#": r"\#",
106 "_": r"\_",
107 "~": r"\textasciitilde{}",
108 "^": r"\textasciicircum{}",
109 '"': "'",
110}
113# U+0085 NEL already falls inside the 0x7F-0x9F range below; only these two
114# separators sit outside it.
115_UNICODE_LINE_SEPARATORS = "\u2028\u2029"
118def is_line_breaking_char(char: str) -> bool:
119 """True if *char* can begin a new line for some consumer of our output.
121 ONE definition, imported by every producer. The recurring bug in this
122 area has been fixing one instance and leaving a sibling — CR handled
123 but not NEL, the URL sanitised but not the citation index — so the
124 union is deliberately wider than any single consumer needs: C0 and DEL
125 for everything, C1 because such a codepoint is never legitimate text,
126 and U+2028/U+2029 because CSS Text makes them mandatory breaks in
127 rendered HTML even though Python's ``re`` and ``str.splitlines()``
128 ignore them.
130 Imported by ``search_utilities._sanitize_sources_field``;
131 ``test_bibliography_escaping`` asserts the two agree on every BMP
132 codepoint, so the claim cannot rot into a comment again.
133 """
134 code = ord(char)
135 return (
136 code < 0x20 or 0x7F <= code <= 0x9F or char in _UNICODE_LINE_SEPARATORS
137 )
140def _escape_bibtex(value: str) -> str:
141 r"""Escape a value for interpolation into a BibTeX field.
143 Titles come from search-result metadata — whatever a page calls itself
144 — and were previously written into ``title = "{...}"`` raw. A title
145 ending the field and opening another (``Benign}", url = "...", note =
146 "{x``) injected an attacker-controlled ``url`` that BibTeX preferred,
147 and an unbalanced brace ran the parser on into the rest of the file.
148 """
149 return "".join(
150 # Control characters are flattened, not escaped. The .bib is
151 # embedded in a ```bibtex markdown fence by the Quarto exporter,
152 # and a bare CR — which ``_BIB_SOURCES_PATTERN``'s ``.`` matches,
153 # since only ``\n`` is excluded — closes that fence once a reader
154 # normalises CRLF, turning the remainder into live markup.
155 # ``_safe_bibtex_url`` already rejects these; this matches it.
156 " " if is_line_breaking_char(ch) else _BIBTEX_ESCAPES.get(ch, ch)
157 for ch in value
158 )
161def _safe_bibtex_url(value: str) -> str:
162 r"""Return *value* if it is safe to place in ``url = {...}``, else "".
164 Escaping is wrong for a URL — a ``\%`` would corrupt a percent-encoded
165 octet. Nor may the offending characters simply be DROPPED: removing a
166 backslash from ``//trusted.example\@evil.example/p`` turns the trusted
167 host into userinfo, so the exported link points at a different host
168 than the one the reader vetted in the report. A URL containing a
169 breakout character is not a URL to repair — it is omitted, and the
170 caller falls through to its existing URL-less branch.
171 """
172 if any(ch in '{}"\\ ' or is_line_breaking_char(ch) for ch in value):
173 # Local-document sources legitimately trip this: a LangChain
174 # retriever sets a result's url from its ``source`` metadata, which
175 # is often a filesystem path carrying spaces or backslashes.
176 # Dropping the link silently would leave the user guessing why a
177 # bibliography entry has none, so record it.
178 # Truncated: a hostile result can carry a megabyte-long URL, and
179 # this fires once per affected entry — and the Quarto exporter
180 # builds the bibliography twice per export, so it fires twice per
181 # entry in practice. ``!r`` keeps control characters escaped.
182 logger.debug(
183 "Omitting unsafe bibliography URL: {!r}{}",
184 value[:200],
185 "…" if len(value) > 200 else "",
186 )
187 return ""
188 return value
191def _iter_citation_group(group: str):
192 r"""Yield each index on one Sources line (``"1, 2, 3"`` -> "1", "2", "3").
194 ``format_links_to_markdown`` collapses every citation of one source
195 onto a single line carrying all of its indices, so a single-index
196 pattern drops such a source from the bibliography entirely. Exporters
197 emit one entry per index so each ``[N]`` in the body resolves.
199 Indices are yielded as STRINGS, exactly as written: the body citations
200 are rewritten from the same text, so converting ``007`` to ``7`` here
201 would key the entry ``ref7`` while the body still cites ``ref007``.
202 No digit filtering either — the pattern's ``\d`` matches only decimal
203 digits (category ``Nd``), which ``int()`` always accepts, so filtering
204 could only discard valid indices such as Arabic-Indic ones.
205 """
206 for part in group.split(","):
207 part = part.strip()
208 if part: 208 ↛ 206line 208 didn't jump to line 206 because the condition on line 208 was always true
209 yield part
212def _sort_key(index: str):
213 """Order bibliography entries numerically where possible."""
214 try:
215 return (0, int(index), "")
216 except ValueError:
217 return (1, 0, index)
220def _collect_bibliography_entries(content: str) -> dict:
221 """Map each citation index to the ``(title, url)`` it names.
223 Selection rule: a URL-bearing entry never loses to a URL-less one, and
224 among URL-bearing entries the LATER occurrence wins.
226 Both halves matter. A report repeats its Sources block once per section
227 and then again as a combined block, and the combined block is both last
228 and authoritative — so later-wins. And the group-aware pattern can match
229 a line of prose that opens with ``[1, 2]`` where a single-index pattern
230 could not; such a line carries no ``URL:``, whereas every line
231 ``format_links_to_markdown`` renders does, so the URL preference is what
232 keeps prose from taking a citation key.
234 Deliberately NOT scoped to a Sources section. Bounding the block at
235 markers like ``\n---`` looked safer but silently emptied the
236 bibliography whenever a thematic break, a setext underline or a table
237 delimiter row came first — while the body was still rewritten to cite
238 entries that no longer existed.
239 """
240 entries: dict[str, tuple[str, str]] = {}
241 for match in _BIB_SOURCES_PATTERN.finditer(content):
242 title = match.group(2).strip()
243 url = match.group(3).strip() if match.group(3) else ""
244 for index in _iter_citation_group(match.group(1)):
245 # Prefer an entry carrying a URL: a real Sources line has one,
246 # a prose false positive does not.
247 if url or index not in entries:
248 entries[index] = (title, url)
249 return entries
252class CitationMode(Enum):
253 """Available citation formatting modes."""
255 NUMBER_HYPERLINKS = "number_hyperlinks" # [1] with hyperlinks
256 DOMAIN_HYPERLINKS = "domain_hyperlinks" # [arxiv.org] with hyperlinks
257 DOMAIN_ID_HYPERLINKS = (
258 "domain_id_hyperlinks" # [arxiv.org] or [arxiv.org-1] with smart IDs
259 )
260 DOMAIN_ID_ALWAYS_HYPERLINKS = (
261 "domain_id_always_hyperlinks" # [arxiv.org-1] always with IDs
262 )
263 SOURCE_TAGGED_HYPERLINKS = "source_tagged_hyperlinks"
264 """Preserve the global citation number and prefix it with a short source
265 tag derived from the URL: known academic sources via ``URLClassifier``
266 (``arxiv-7``, ``pubmed-3``), domain otherwise (``nytimes.com-9``), and
267 ``local-N`` for empty / local URLs. Unlike DOMAIN_ID_* modes the
268 suffix is the original citation number, so labels never collide and
269 match the bibliography order: ``[1]`` arxiv + ``[2]`` openai + ``[3]``
270 arxiv -> ``arxiv-1``, ``openai-2``, ``arxiv-3``."""
271 NO_HYPERLINKS = "no_hyperlinks" # [1] without hyperlinks
274class CitationFormatter:
275 """Formats citations in markdown documents with various styles."""
277 def __init__(self, mode: CitationMode = CitationMode.NUMBER_HYPERLINKS):
278 self.mode = mode
279 # Use negative lookbehind and lookahead to avoid matching already formatted citations
280 # Also match Unicode lenticular brackets 【】 (U+3010 and U+3011) that LLMs sometimes generate
281 self.citation_pattern = re.compile(
282 r"(?<![\[【])[\[【](\d+)[\]】](?![\]】])"
283 )
284 self.comma_citation_pattern = re.compile(
285 r"[\[【](\d+(?:,\s*\d+)+)[\]】]"
286 )
287 # Also match "Source X" or "source X" patterns
288 self.source_word_pattern = re.compile(r"\b[Ss]ource\s+(\d+)\b")
289 self.sources_pattern = re.compile(
290 r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
291 re.MULTILINE,
292 )
294 def _create_source_word_replacer(self, formatter_func):
295 """Create a replacement function for 'Source X' patterns.
297 Args:
298 formatter_func: A function that takes citation_num and returns formatted text
300 Returns:
301 A replacement function for use with regex sub
302 """
304 def replace_source_word(match):
305 citation_num = match.group(1)
306 return formatter_func(citation_num)
308 return replace_source_word
310 def _create_citation_formatter(self, sources_dict, format_pattern):
311 """Create a formatter function for citations.
313 Args:
314 sources_dict: Dictionary mapping citation numbers to data
315 format_pattern: A callable that takes (citation_num, data) and returns formatted string
317 Returns:
318 A function that formats citations or returns fallback
319 """
321 def formatter(citation_num):
322 if citation_num in sources_dict:
323 data = sources_dict[citation_num]
324 return format_pattern(citation_num, data)
325 return f"[{citation_num}]"
327 return formatter
329 def _replace_comma_citations(self, content, lookup, format_one):
330 """Replace comma-separated citations like [1, 2, 3] using *lookup* and *format_one*.
332 Args:
333 content: Text to process
334 lookup: Dict mapping citation number (str) to data
335 format_one: ``(num, data) -> str`` callback that formats a single citation
336 """
338 def _replacer(match):
339 nums = [n.strip() for n in match.group(1).split(",")]
340 parts = []
341 for num in nums:
342 if num in lookup:
343 parts.append(format_one(num, lookup[num]))
344 else:
345 parts.append(f"[{num}]")
346 return "".join(parts)
348 return self.comma_citation_pattern.sub(_replacer, content)
350 def format_document(self, content: str) -> str:
351 """Format citations in *content* and return the full document.
353 This is the legacy single-string entry point: it concatenates
354 the answer half and the sources half produced by
355 :meth:`format_document_split`. New code that needs to know
356 where the boundary between answer and sources actually fell
357 should use :meth:`format_document_split` instead so the boundary
358 is returned explicitly (no re-parsing of the concatenated output).
359 """
360 formatted_answer, sources_md, _on_sentinel = self.format_document_split(
361 content
362 )
363 return formatted_answer + sources_md
365 def format_document_split(
366 self, content: str, *, trust_sentinel: bool = True
367 ) -> Tuple[str, str, bool]:
368 """Format citations and return ``(answer, sources_md, on_sentinel)``.
370 The boundary between the LLM's answer and the trailing Sources
371 section is computed inside this method. Callers that only want
372 the answer (e.g. the chat-mode save site) get a clean split
373 without re-applying a regex on concatenated output downstream.
374 ``on_sentinel`` mirrors the second element of
375 :func:`find_sources_section`'s return — ``True`` iff the split
376 boundary came from the appended-sources sentinel emitted by
377 ``IntegratedReportGenerator``, in which case the boundary is
378 correct by construction and downstream over-strip safety checks
379 should be skipped. Pass ``trust_sentinel=False`` for raw LLM
380 output (quick-mode save path) where the sentinel would only
381 appear if the LLM quoted the marker itself.
383 Returns ``(content, "", False)`` when the formatter is in
384 NO_HYPERLINKS mode or when no Sources section can be found in
385 ``content``.
386 """
387 if self.mode == CitationMode.NO_HYPERLINKS:
388 return content, "", False
390 sources_start, on_sentinel = find_sources_section(
391 content, trust_sentinel=trust_sentinel
392 )
393 if sources_start == -1:
394 return content, "", False
396 document_content = content[:sources_start]
397 sources_content = content[sources_start:]
399 sources = self._parse_sources(sources_content)
401 if self.mode == CitationMode.NUMBER_HYPERLINKS:
402 formatted_content = self._format_number_hyperlinks(
403 document_content, sources
404 )
405 elif self.mode == CitationMode.DOMAIN_HYPERLINKS:
406 formatted_content = self._format_domain_hyperlinks(
407 document_content, sources
408 )
409 elif self.mode == CitationMode.DOMAIN_ID_HYPERLINKS:
410 formatted_content = self._format_domain_id_hyperlinks(
411 document_content, sources
412 )
413 elif self.mode == CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS:
414 formatted_content = self._format_domain_id_always_hyperlinks(
415 document_content, sources
416 )
417 elif self.mode == CitationMode.SOURCE_TAGGED_HYPERLINKS: 417 ↛ 424line 417 didn't jump to line 424 because the condition on line 417 was always true
418 formatted_content = self._format_source_tagged_hyperlinks(
419 document_content,
420 sources,
421 self._parse_collections(sources_content),
422 )
423 else:
424 formatted_content = document_content
426 return formatted_content, sources_content, on_sentinel
428 def apply_inline_hyperlinks(
429 self, content: str, sources: List[Dict[str, Any]]
430 ) -> str:
431 """Hyperlink ``[N]`` refs using a structured source list.
433 Dispatches on ``self.mode`` so the user's chosen citation
434 format (Settings → Report → Citation Format) is honored on
435 the fallback path the same way it is in
436 :meth:`format_document_split`. Inherits all the existing
437 per-mode guards (lookbehind/lookahead against ``[[1]]``,
438 comma-list handling like ``[1,2,3]``, ``Source N`` word form,
439 missing-index pass-through, lenticular bracket support).
441 Used as the safe fallback at save time when the LLM does NOT
442 emit a Sources section in its prose — the structured source
443 list (e.g. ``search_system.all_links_of_system``) is the
444 canonical source of URLs and indices.
445 """
446 if not content or not sources:
447 return content or ""
448 if self.mode == CitationMode.NO_HYPERLINKS:
449 return content
451 # Search-engine result dicts use either "url" or "link" for the
452 # destination — Searxng emits {"link": ..., "title": ..., "snippet": ...}
453 # (search_engine_searxng.py:538) and other engines use "url".
454 # Looking up only `s["url"]` silently dropped every Searxng-sourced
455 # citation, leaving the answer body with plain `[N]` brackets even
456 # though the Sources section beneath was fully populated. Accept
457 # both keys so the hyperlink fallback works regardless of engine.
458 def _src_url(s):
459 return s.get("url") or s.get("link") or ""
461 adapted: Dict[str, Tuple[str, str]] = {
462 str(s["index"]): (s.get("title", "Untitled"), _src_url(s))
463 for s in sources
464 if _src_url(s) and s.get("index") is not None
465 }
466 if not adapted:
467 return content
469 # Per-mode dispatch — mirrors format_document_split so the user's
470 # chosen citation format applies on this fallback path too.
471 # Previously this was hard-coded to _format_number_hyperlinks,
472 # which meant chat-mode answers (which always hit this fallback
473 # because the langgraph-agent synthesis doesn't emit a ## Sources
474 # block in its prose) ignored the report.citation_format setting
475 # entirely — every chat answer came out as [[N]](url) even when
476 # the user picked domain-based or source-tagged formatting.
477 if self.mode == CitationMode.DOMAIN_HYPERLINKS:
478 return self._format_domain_hyperlinks(content, adapted)
479 if self.mode == CitationMode.DOMAIN_ID_HYPERLINKS:
480 return self._format_domain_id_hyperlinks(content, adapted)
481 if self.mode == CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS: 481 ↛ 482line 481 didn't jump to line 482 because the condition on line 481 was never true
482 return self._format_domain_id_always_hyperlinks(content, adapted)
483 if self.mode == CitationMode.SOURCE_TAGGED_HYPERLINKS:
484 # Pull collection names off the structured source dicts
485 # (format_links_to_markdown uses the same shape:
486 # link["metadata"]["collection_name"]) so the SOURCE_TAGGED
487 # formatter can surface library/RAG tags as the citation
488 # label when present.
489 collections: Dict[str, str] = {}
490 for s in sources:
491 idx = s.get("index")
492 if idx is None: 492 ↛ 493line 492 didn't jump to line 493 because the condition on line 492 was never true
493 continue
494 meta = s.get("metadata") or {}
495 coll = meta.get("collection_name")
496 if coll:
497 collections.setdefault(str(idx), str(coll))
498 return self._format_source_tagged_hyperlinks(
499 content, adapted, collections
500 )
501 # NUMBER_HYPERLINKS is the default and the catch-all for any
502 # mode added later that doesn't have an explicit branch above.
503 return self._format_number_hyperlinks(content, adapted)
505 def _find_sources_section(
506 self, content: str, trust_sentinel: bool = True
507 ) -> Tuple[int, bool]:
508 """Find the start of the sources/references section.
510 Returns ``(position, on_sentinel)`` — see
511 :func:`find_sources_section` for the semantics. ``on_sentinel``
512 is True iff the boundary came from the unique sentinel emitted
513 by the report generator (i.e. it is correct by construction).
514 """
515 return find_sources_section(content, trust_sentinel=trust_sentinel)
517 def _parse_sources(
518 self, sources_content: str
519 ) -> Dict[str, Tuple[str, str]]:
520 """
521 Parse sources section to extract citation numbers, titles, and URLs.
523 Returns:
524 Dictionary mapping citation number to (title, url) tuple
525 """
526 sources = {}
527 matches = list(self.sources_pattern.finditer(sources_content))
529 for match in matches:
530 citation_nums_str = match.group(1)
531 title = match.group(2).strip()
532 url = match.group(3).strip() if match.group(3) else ""
534 # Handle comma-separated citation numbers like [36, 3]
535 # Split by comma and strip whitespace
536 individual_nums = [
537 num.strip() for num in citation_nums_str.split(",")
538 ]
540 # Add an entry for each individual number
541 for num in individual_nums:
542 sources[num] = (title, url)
544 return sources
546 def _format_number_hyperlinks(
547 self, content: str, sources: Dict[str, Tuple[str, str]]
548 ) -> str:
549 """Replace [1] with hyperlinked version where only the number is linked."""
550 # Filter sources that have URLs
551 url_sources = {
552 num: (title, url) for num, (title, url) in sources.items() if url
553 }
555 # Create formatter for citations with number hyperlinks
556 def format_number_link(citation_num, data):
557 _, url = data
558 return f"[[{citation_num}]]({url})"
560 # Handle comma-separated citations like [1, 2, 3]
561 content = self._replace_comma_citations(
562 content, url_sources, format_number_link
563 )
565 formatter = self._create_citation_formatter(
566 url_sources, format_number_link
567 )
569 # Handle individual citations
570 def replace_citation(match):
571 return (
572 formatter(match.group(1))
573 if match.group(1) in url_sources
574 else match.group(0)
575 )
577 content = self.citation_pattern.sub(replace_citation, content)
579 # Also handle "Source X" patterns
580 return self.source_word_pattern.sub(
581 self._create_source_word_replacer(formatter), content
582 )
584 def _format_domain_hyperlinks(
585 self, content: str, sources: Dict[str, Tuple[str, str]]
586 ) -> str:
587 """Replace [1] with [domain.com] hyperlinked version."""
589 # Filter sources that have URLs
590 url_sources = {
591 num: (title, url) for num, (title, url) in sources.items() if url
592 }
594 # Pre-compute labels so [1, 2, 3] lists and standalone [N] refs
595 # use the same string for a given citation. Previously this
596 # closure called _extract_domain on every match, so a relative
597 # URL (``/library/document/...``) produced ``[[]]`` for every
598 # occurrence. _citation_label falls back to a slugified title
599 # for relative URLs, keeping the label non-empty and readable.
600 label_cache: Dict[str, str] = {
601 num: self._citation_label(num, title, url)
602 for num, (title, url) in url_sources.items()
603 }
605 # Create formatter for citations with domain hyperlinks
606 def format_domain_link(citation_num, data):
607 _, url = data
608 label = label_cache[citation_num]
609 return f"[[{label}]]({url})"
611 # Handle comma-separated citations like [1, 2, 3]
612 content = self._replace_comma_citations(
613 content, url_sources, format_domain_link
614 )
616 formatter = self._create_citation_formatter(
617 url_sources, format_domain_link
618 )
620 # Handle individual citations
621 def replace_citation(match):
622 return (
623 formatter(match.group(1))
624 if match.group(1) in url_sources
625 else match.group(0)
626 )
628 content = self.citation_pattern.sub(replace_citation, content)
630 # Also handle "Source X" patterns
631 return self.source_word_pattern.sub(
632 self._create_source_word_replacer(formatter), content
633 )
635 def _format_domain_id_hyperlinks(
636 self, content: str, sources: Dict[str, Tuple[str, str]]
637 ) -> str:
638 """Replace [1] with [domain.com-1] hyperlinked version with hyphen-separated IDs."""
639 # First, create a mapping of domains to their citation numbers.
640 # _citation_label (vs raw _extract_domain) means relative URLs
641 # like ``/library/document/...`` get a slugified title as the
642 # "domain" instead of an empty string. Without this, a RAG
643 # result with one citation rendered as ``[[]](url)`` (the
644 # label disappeared), and a multi-citation set rendered as
645 # ``[[-1]](url)`` (the orphaned ``-N`` suffix leaked through).
646 domain_citations: dict[str, list[Any]] = {}
648 for citation_num, (title, url) in sources.items():
649 if url:
650 domain = self._citation_label(citation_num, title, url)
651 if domain not in domain_citations:
652 domain_citations[domain] = []
653 domain_citations[domain].append((citation_num, url))
655 # Create a mapping from citation number to domain with ID
656 citation_to_domain_id = {}
657 for domain, citations in domain_citations.items():
658 if len(citations) > 1:
659 # Multiple citations from same domain - add hyphen and number
660 for idx, (citation_num, url) in enumerate(citations, 1):
661 citation_to_domain_id[citation_num] = (
662 f"{domain}-{idx}",
663 url,
664 )
665 else:
666 # Single citation from domain - no ID needed
667 citation_num, url = citations[0]
668 citation_to_domain_id[citation_num] = (domain, url)
670 # Create formatter for citations with domain_id hyperlinks
671 def format_domain_id_link(citation_num, data):
672 domain_id, url = data
673 return f"[[{domain_id}]]({url})"
675 # Handle comma-separated citations
676 content = self._replace_comma_citations(
677 content, citation_to_domain_id, format_domain_id_link
678 )
680 formatter = self._create_citation_formatter(
681 citation_to_domain_id, format_domain_id_link
682 )
684 # Handle individual citations
685 def replace_citation(match):
686 return (
687 formatter(match.group(1))
688 if match.group(1) in citation_to_domain_id
689 else match.group(0)
690 )
692 content = self.citation_pattern.sub(replace_citation, content)
694 # Also handle "Source X" patterns
695 return self.source_word_pattern.sub(
696 self._create_source_word_replacer(formatter), content
697 )
699 def _format_domain_id_always_hyperlinks(
700 self, content: str, sources: Dict[str, Tuple[str, str]]
701 ) -> str:
702 """Replace [1] with [domain.com-1] hyperlinked version, always with IDs."""
703 # Use _citation_label so relative URLs (e.g. RAG /library/document/...
704 # results) produce a slugified-title prefix instead of an empty
705 # string. Without this, every label here collapses to ``-N``
706 # (e.g. ``[[-1]](url)``) and the user has no idea what the link
707 # references. See _format_domain_id_hyperlinks for the same fix.
708 domain_citations: dict[str, list[Any]] = {}
710 for citation_num, (title, url) in sources.items():
711 if url:
712 domain = self._citation_label(citation_num, title, url)
713 if domain not in domain_citations:
714 domain_citations[domain] = []
715 domain_citations[domain].append((citation_num, url))
717 # Create a mapping from citation number to domain with ID
718 citation_to_domain_id = {}
719 for domain, citations in domain_citations.items():
720 # Always add hyphen and number for consistency
721 for idx, (citation_num, url) in enumerate(citations, 1):
722 citation_to_domain_id[citation_num] = (f"{domain}-{idx}", url)
724 # Create formatter for citations with domain_id hyperlinks
725 def format_domain_id_link(citation_num, data):
726 domain_id, url = data
727 return f"[[{domain_id}]]({url})"
729 # Handle comma-separated citations
730 content = self._replace_comma_citations(
731 content, citation_to_domain_id, format_domain_id_link
732 )
734 formatter = self._create_citation_formatter(
735 citation_to_domain_id, format_domain_id_link
736 )
738 # Handle individual citations
739 def replace_citation(match):
740 return (
741 formatter(match.group(1))
742 if match.group(1) in citation_to_domain_id
743 else match.group(0)
744 )
746 content = self.citation_pattern.sub(replace_citation, content)
748 # Also handle "Source X" patterns
749 return self.source_word_pattern.sub(
750 self._create_source_word_replacer(formatter), content
751 )
753 # Sources section may carry a "Collection: <name>" line for RAG /
754 # library hits (emitted by ``utilities/search_utilities.format_links_to_markdown``).
755 # The line sits between this ``[N]`` entry's ``URL:`` line and the
756 # next ``[N+1]`` entry. We anchor the match on a non-greedy span up
757 # to the next citation header (or end of string) to scope correctly.
758 _collection_line_pattern = re.compile(
759 r"^\[(\d+(?:,\s*\d+)*)\][^\n]*\n" # the [N] header line
760 r"(?:[^\n\[]*\n)*?" # any non-[ lines (typically URL: ...)
761 r"\s*Collection:\s*(.+?)\s*$",
762 re.MULTILINE,
763 )
765 def _parse_collections(self, sources_content: str) -> Dict[str, str]:
766 """Extract ``{citation_num: collection_name}`` from a sources
767 block. Returns an empty dict when no ``Collection:`` lines exist
768 — the absence of collection info is the common case (web URLs)
769 and must never raise."""
770 collections: Dict[str, str] = {}
771 for match in self._collection_line_pattern.finditer(sources_content):
772 citation_nums_str = match.group(1)
773 collection = match.group(2).strip()
774 if not collection: 774 ↛ 775line 774 didn't jump to line 775 because the condition on line 774 was never true
775 continue
776 for num in (n.strip() for n in citation_nums_str.split(",")):
777 collections[num] = collection
778 return collections
780 def _format_source_tagged_hyperlinks(
781 self,
782 content: str,
783 sources: Dict[str, Tuple[str, str]],
784 collections: Dict[str, str],
785 ) -> str:
786 """Replace ``[N]`` with ``[[source-N]](url)``.
788 ``source`` resolves to (in order): the RAG ``Collection:``
789 tag for library hits, the short URLClassifier tag for known
790 academic sources (``arxiv``, ``pubmed``, ...), the cleaned
791 domain otherwise, or ``local`` for empty/file URLs. ``N`` is
792 the original global citation number — labels never collide and
793 the suffix always matches the bibliography ordering.
795 Args:
796 content: Document body (sources section already split off).
797 sources: ``{citation_num: (title, url)}`` parsed from the
798 sources block.
799 collections: ``{citation_num: collection_name}`` parsed from
800 optional ``Collection:`` lines in the sources block
801 (empty dict when no library/RAG hits are cited). Wins
802 over URL-derived tags when present for a given citation.
803 """
805 # Pre-compute tags so each URL-having source resolves its label
806 # once, however many [N] / [1, 2, 3] refs cite it.
807 tag_cache: Dict[str, str] = {
808 num: self._extract_source_label(
809 url, collection=collections.get(num)
810 )
811 for num, (_, url) in sources.items()
812 if url
813 }
815 def format_link(citation_num, data):
816 _, url = data
817 label = tag_cache.get(citation_num)
818 if label is None:
819 # Only URL-less sources are absent from tag_cache. Don't
820 # pass this as dict.get()'s default — that would evaluate
821 # the expensive lookup eagerly on every cache hit too.
822 label = self._extract_source_label(
823 url, collection=collections.get(citation_num)
824 )
825 tag = f"{label}-{citation_num}"
826 # Only emit a hyperlink for http(s) URLs — local/file URLs are
827 # rendered as plain bracketed tags so the markdown stays clean
828 # and viewers don't try to navigate to a server-local path.
829 return (
830 f"[[{tag}]]({url})"
831 if self._is_linkable_url(url)
832 else f"[{tag}]"
833 )
835 # Handle comma-separated citations like [1, 2, 3]
836 content = self._replace_comma_citations(content, sources, format_link)
838 formatter = self._create_citation_formatter(sources, format_link)
840 # Handle individual citations
841 def replace_citation(match):
842 return (
843 formatter(match.group(1))
844 if match.group(1) in sources
845 else match.group(0)
846 )
848 content = self.citation_pattern.sub(replace_citation, content)
850 # Also handle "Source X" patterns
851 return self.source_word_pattern.sub(
852 self._create_source_word_replacer(formatter), content
853 )
855 @staticmethod
856 def _slugify_collection(name: str) -> str:
857 """Make a user-set collection name safe for inline citations.
859 Collection names are free-form strings (``"My Papers"``,
860 ``"team/finance"``). Citations need a compact token that won't
861 break markdown — strip whitespace, lowercase, replace runs of
862 non-alphanumeric chars with a single hyphen, trim leading and
863 trailing hyphens, and fall back to ``"local"`` if the result is
864 empty. ``-N`` is appended downstream so we strip trailing
865 hyphens to keep the join clean.
866 """
867 slug = re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-")
868 return slug or "local"
870 @staticmethod
871 def _slugify_title(title: str, max_length: int = 32) -> Optional[str]:
872 """Generate a slug from a document title using python-slugify.
874 Returns ``None`` when ``title`` is empty / whitespace / pure
875 punctuation, or when ``slugify`` produces an empty result. The
876 ``None`` sentinel lets callers distinguish "no meaningful slug"
877 from "slug happens to be a real word" (e.g. a document literally
878 titled ``"Doc"`` must yield ``"doc"`` as its label, not fall
879 through to the citation-number fallback).
880 """
881 if not title:
882 return None
883 slug = slugify(title, max_length=max_length, separator="-")
884 return slug if slug else None
886 def _citation_label(self, citation_num: str, title: str, url: str) -> str:
887 """Return the inline-citation label for a source.
889 Order of preference:
890 1. The URL's domain (``arxiv.org``, ``example.com``) — used by
891 web citations. Falls through when the URL is relative or
892 empty, as is common for RAG / library documents.
893 2. A slugified version of the document title (truncated to
894 keep labels compact) — gives users a readable label for
895 local-library hits.
896 3. The citation number itself — last-resort guarantee the
897 label is never empty (an empty label produces ``[[]]``
898 which renders as an uninformative ``[]`` link).
899 """
900 domain = self._extract_domain(url) if url else ""
901 if domain:
902 return domain
903 slug = self._slugify_title(title)
904 if slug:
905 return slug
906 return str(citation_num)
908 @staticmethod
909 def _is_linkable_url(url: str) -> bool:
910 """Return True iff ``url`` is a http(s) URL safe to wrap in a
911 markdown hyperlink. Empty strings and file:// / local: schemes
912 are not linkable."""
913 if not url:
914 return False
915 try:
916 scheme = (urlparse(url).scheme or "").lower()
917 except (ValueError, AttributeError):
918 return False
919 return scheme in ("http", "https")
921 def _extract_source_label(
922 self, url: str, collection: str | None = None
923 ) -> str:
924 """Return a short source tag for ``url``.
926 Resolution order:
927 1. ``collection`` (when supplied) wins outright — RAG / library
928 hits surface their collection name as the citation tag
929 (``mypapers``, ``personal-notes``, ...). The renderer in
930 ``utilities/search_utilities.format_links_to_markdown``
931 emits a ``Collection:`` line per source for library results,
932 which the formatter parses back into this argument.
933 2. Empty URL or non-http(s) scheme (``file://``, ``local:``, ...) →
934 ``"local"``. Uniform fallback when no collection name is
935 available.
936 3. ``URLClassifier`` matches a known academic source → use the
937 enum value (``arxiv``, ``pubmed``, ``pmc``, ``biorxiv``,
938 ``medrxiv``, ``semantic_scholar``, ``doi``).
939 4. Otherwise → fall back to ``_extract_domain`` (e.g.
940 ``arxiv.org``, ``nytimes.com``).
941 """
942 if collection:
943 return self._slugify_collection(collection)
944 if not url:
945 return "local"
946 try:
947 parsed = urlparse(url)
948 except (ValueError, AttributeError):
949 return "local"
950 scheme = (parsed.scheme or "").lower()
951 if scheme not in ("http", "https"):
952 return "local"
954 url_type = URLClassifier.classify(url)
955 # Generic HTML/PDF/INVALID → fall back to domain. Everything else
956 # is a known academic source whose enum value is the short tag.
957 if url_type in (URLType.HTML, URLType.PDF, URLType.INVALID):
958 return self._extract_domain(url)
959 return str(url_type.value)
961 def _extract_domain(self, url: str) -> str:
962 """Extract domain name from URL."""
963 try:
964 parsed = urlparse(url)
965 domain = parsed.netloc
966 # Remove www. prefix if present
967 if domain.startswith("www."):
968 domain = domain[4:]
969 # Keep known domains as-is
970 known_domains = {
971 "arxiv.org": "arxiv.org",
972 "github.com": "github.com",
973 "reddit.com": "reddit.com",
974 "youtube.com": "youtube.com",
975 "pypi.org": "pypi.org",
976 "milvus.io": "milvus.io",
977 "medium.com": "medium.com",
978 }
980 for known, display in known_domains.items():
981 if known in domain:
982 return display
984 # For other domains, extract main domain
985 parts = domain.split(".")
986 if len(parts) >= 2:
987 return ".".join(parts[-2:])
988 return domain
989 except (ValueError, AttributeError):
990 return "source"
993class QuartoExporter:
994 """Export markdown documents to Quarto (.qmd) format."""
996 def __init__(self):
997 # Also match Unicode lenticular brackets 【】 (U+3010 and U+3011) that LLMs sometimes generate
998 self.citation_pattern = re.compile(
999 r"(?<![\[【])[\[【](\d+)[\]】](?![\]】])"
1000 )
1001 self.comma_citation_pattern = re.compile(
1002 r"[\[【](\d+(?:,\s*\d+)+)[\]】]"
1003 )
1005 def export_to_quarto(self, content: str, title: str | None = None) -> str:
1006 """
1007 Convert markdown document to Quarto format.
1009 Args:
1010 content: Markdown content
1011 title: Document title (if None, will extract from content)
1013 Returns:
1014 Quarto formatted content
1015 """
1016 # Extract title from markdown if not provided
1017 if not title:
1018 title_match = re.search(r"^#\s+(.+)$", content, re.MULTILINE)
1019 title = title_match.group(1) if title_match else "Research Report"
1021 # Create Quarto YAML header
1022 from datetime import UTC, datetime
1024 current_date = datetime.now(UTC).strftime("%Y-%m-%d")
1025 yaml_header = f"""---
1026title: "{title}"
1027author: "Local Deep Research"
1028date: "{current_date}"
1029format:
1030 html:
1031 toc: true
1032 toc-depth: 3
1033 number-sections: true
1034 pdf:
1035 toc: true
1036 number-sections: true
1037 colorlinks: true
1038bibliography: references.bib
1039csl: apa.csl
1040---
1042"""
1044 # Process content
1045 processed_content = content
1047 # First handle comma-separated citations like [1, 2, 3]
1048 def replace_comma_citations(match):
1049 citation_nums = match.group(1)
1050 # Split by comma and strip whitespace
1051 nums = [num.strip() for num in citation_nums.split(",")]
1052 refs = [f"@ref{num}" for num in nums]
1053 return f"[{', '.join(refs)}]"
1055 processed_content = self.comma_citation_pattern.sub(
1056 replace_comma_citations, processed_content
1057 )
1059 # Then convert individual citations to Quarto format [@citation]
1060 def replace_citation(match):
1061 citation_num = match.group(1)
1062 return f"[@ref{citation_num}]"
1064 processed_content = self.citation_pattern.sub(
1065 replace_citation, processed_content
1066 )
1068 # Generate bibliography file content
1069 bib_content = self._generate_bibliography(content)
1071 # Add note about bibliography file
1072 bibliography_note = (
1073 "\n\n::: {.callout-note}\n## Bibliography File Required\n\nThis document requires a `references.bib` file in the same directory with the following content:\n\n```bibtex\n"
1074 + bib_content
1075 + "\n```\n:::\n"
1076 )
1078 return yaml_header + processed_content + bibliography_note
1080 def _generate_bibliography(self, content: str) -> str:
1081 """Generate BibTeX bibliography from sources."""
1082 entries = _collect_bibliography_entries(content)
1084 bibliography = ""
1085 for citation_num in sorted(entries, key=_sort_key):
1086 title, url = entries[citation_num]
1087 safe_title = _escape_bibtex(title)
1088 safe_url = _safe_bibtex_url(url)
1089 bib_entry = f"@misc{{ref{citation_num},\n"
1090 bib_entry += f' title = "{{{safe_title}}}",\n'
1091 if safe_url:
1092 bib_entry += f" url = {{{safe_url}}},\n"
1093 bib_entry += f' howpublished = "\\url{{{safe_url}}}",\n'
1094 bib_entry += f" year = {{{2024}}},\n"
1095 bib_entry += ' note = "Accessed: \\today"\n'
1096 bib_entry += "}\n"
1098 bibliography += bib_entry + "\n"
1100 return bibliography.strip()
1103class RISExporter:
1104 """Export references to RIS format for reference managers like Zotero."""
1106 def __init__(self):
1107 self.sources_pattern = re.compile(
1108 r"^\[(\d+(?:,\s*\d+)*)\]\s*(.+?)(?:\n\s*URL:\s*(.+?))?$",
1109 re.MULTILINE,
1110 )
1112 def export_to_ris(self, content: str) -> str:
1113 """
1114 Extract references from markdown and convert to RIS format.
1116 Args:
1117 content: Markdown content with sources
1119 Returns:
1120 RIS formatted references
1121 """
1122 # Find sources section
1123 sources_start, _on_sentinel = find_sources_section(content)
1124 if sources_start == -1:
1125 return ""
1127 # Find the end of the first sources section (before any other major section)
1128 sources_content = content[sources_start:]
1130 # Look for the next major section to avoid duplicates
1131 next_section_markers = [
1132 "\n## ALL SOURCES",
1133 "\n### ALL SOURCES",
1134 "\n## Research Metrics",
1135 "\n### Research Metrics",
1136 "\n## SEARCH QUESTIONS",
1137 "\n### SEARCH QUESTIONS",
1138 "\n## DETAILED FINDINGS",
1139 "\n### DETAILED FINDINGS",
1140 "\n---", # Horizontal rule often separates sections
1141 ]
1143 sources_end = len(sources_content)
1144 for marker in next_section_markers:
1145 pos = sources_content.find(marker)
1146 if pos != -1 and pos < sources_end:
1147 sources_end = pos
1149 sources_content = sources_content[:sources_end]
1151 # Parse sources and generate RIS entries
1152 ris_entries = []
1153 seen_refs = set() # Track which references we've already processed
1155 # Split sources into individual entries
1156 import re
1158 # Pattern to match each source entry. The opener is GROUP-AWARE
1159 # (``\d+(?:,\s*\d+)*``, the spelling used by
1160 # ``comma_citation_pattern`` and ``_BIB_SOURCES_PATTERN`` above):
1161 # ``format_links_to_markdown`` collapses every citation of one source
1162 # onto a single merged ``[1, 3] Title`` line, which is routine for
1163 # library documents since per-document keying (#5685). A single-index
1164 # ``(\d+)`` capture did not merely drop such a line — it MIS-CITED
1165 # (#5687), because the lookahead did not terminate at a merged opener
1166 # either, so the PRECEDING entry swallowed the merged line and the
1167 # last-wins ``URL:`` scan below then handed that entry the merged
1168 # line's URL: one source's title against another source's link. Hence
1169 # the group spelling appears TWICE — once in the capture, once in the
1170 # lookahead — and the loop emits one record PER INDEX so that every
1171 # ``[N]`` in the body resolves to a reference of its own.
1172 #
1173 # Four properties here are deliberate; keep them when editing:
1174 # 1. This parser stays bounded to the FIRST Sources section by the
1175 # ``next_section_markers`` scan above. Do not swap in the
1176 # module-level ``_collect_bibliography_entries`` — that helper
1177 # is deliberately not section-scoped, which is why #5687 notes
1178 # RIS is not a mechanical swap even though BibTeX, Quarto and
1179 # LaTeX were fixed by adopting it.
1180 # 2. Lenticular "【N】" openers/closers are accepted alongside
1181 # ASCII "[N]": the inline citation patterns in this file already
1182 # handle them (some LLMs emit them), so the source-list parser
1183 # must stay consistent or it would silently drop those entries.
1184 # 3. Entries are split by LOOKAHEAD under ``DOTALL`` so continuation
1185 # lines (``URL:``, ``DOI:``, ``Published in``) stay attached to
1186 # the entry they belong to; a line-anchored ``$`` split would
1187 # strip them.
1188 # 4. ``self.sources_pattern`` above is group-aware but UNUSED, and
1189 # is not this fix — it is single-line and section-unaware, so
1190 # wiring it in would lose properties 1 and 3.
1191 source_entry_pattern = re.compile(
1192 r"^[\[【](\d+(?:,\s*\d+)*)[\]】]\s*(.+?)"
1193 r"(?=^[\[【]\d+(?:,\s*\d+)*[\]】]|\Z)",
1194 re.MULTILINE | re.DOTALL,
1195 )
1197 for match in source_entry_pattern.finditer(sources_content):
1198 citation_group = match.group(1)
1199 entry_text = match.group(2).strip()
1201 # Extract the title (first line)
1202 lines = entry_text.split("\n")
1203 title = lines[0].strip()
1205 # Extract URL, DOI, and other metadata from subsequent lines
1206 url = ""
1207 metadata = {}
1208 for line in lines[1:]:
1209 line = line.strip()
1210 if line.startswith("URL:"):
1211 url = line[4:].strip()
1212 elif line.startswith("DOI:"):
1213 metadata["doi"] = line[4:].strip()
1214 elif line.startswith("Published in"):
1215 metadata["journal"] = line[12:].strip()
1216 # Add more metadata parsing as needed
1217 elif line:
1218 # Store other lines as additional metadata
1219 if "additional" not in metadata: 1219 ↛ 1221line 1219 didn't jump to line 1221 because the condition on line 1219 was always true
1220 metadata["additional"] = []
1221 additional = metadata["additional"]
1222 if isinstance(additional, list): 1222 ↛ 1208line 1222 didn't jump to line 1208 because the condition on line 1222 was always true
1223 additional.append(line)
1225 # Combine title with additional metadata lines for full context
1226 full_text = entry_text
1228 # One record per index, keyed on the INDIVIDUAL index rather than
1229 # the group text: a report repeats its Sources block, and the same
1230 # source may be written ``[1]`` in one block and ``[1, 3]`` in
1231 # another, so keying on the raw group would let ``ref1`` be emitted
1232 # twice. ``metadata`` is shared across the indices of one line;
1233 # ``_create_ris_entry`` only reads it.
1234 for citation_num in _iter_citation_group(citation_group):
1235 ref_key = (citation_num, title, url)
1236 if ref_key not in seen_refs:
1237 seen_refs.add(ref_key)
1238 # Create RIS entry with full text for metadata extraction
1239 ris_entry = self._create_ris_entry(
1240 citation_num, full_text, url, metadata
1241 )
1242 ris_entries.append(ris_entry)
1244 return "\n".join(ris_entries)
1246 def _create_ris_entry(
1247 self,
1248 ref_id: str,
1249 full_text: str,
1250 url: str = "",
1251 metadata: dict | None = None,
1252 ) -> str:
1253 """Create a single RIS entry."""
1254 lines = []
1256 # Parse metadata from full text
1257 import re
1259 if metadata is None:
1260 metadata = {}
1262 # Extract title from first line. NB: split into a *separate* variable —
1263 # ``lines`` is the RIS-output accumulator initialized above and appended
1264 # to below; reusing it here previously overwrote it with the source
1265 # text, so every entry emitted the raw source body before the mandatory
1266 # leading ``TY - `` tag and reference managers rejected the file.
1267 text_lines = full_text.split("\n")
1268 title = text_lines[0].strip()
1270 # Extract year from full text (looks for 4-digit year)
1271 year_match = re.search(r"\b(19\d{2}|20\d{2})\b", full_text)
1272 year = year_match.group(1) if year_match else None
1274 # Extract authors if present (looks for "by Author1, Author2")
1275 authors_match = re.search(
1276 r"\bby\s+([^.\n]+?)(?:\.|\n|$)", full_text, re.IGNORECASE
1277 )
1278 authors = []
1279 if authors_match:
1280 authors_text = authors_match.group(1)
1281 # Split by 'and' or ','
1282 author_parts = re.split(r"\s*(?:,|\sand\s|&)\s*", authors_text)
1283 authors = [a.strip() for a in author_parts if a.strip()]
1285 # Extract DOI from metadata or text
1286 doi = metadata.get("doi")
1287 if not doi:
1288 doi_match = re.search(
1289 r"DOI:\s*([^\s\n]+)", full_text, re.IGNORECASE
1290 )
1291 doi = doi_match.group(1) if doi_match else None
1293 # Clean title - remove author and metadata info for cleaner title
1294 clean_title = title
1295 if authors_match and authors_match.start() < len(title):
1296 clean_title = (
1297 title[: authors_match.start()] + title[authors_match.end() :]
1298 if authors_match.end() < len(title)
1299 else title[: authors_match.start()]
1300 )
1301 clean_title = re.sub(
1302 r"\s*DOI:\s*[^\s]+", "", clean_title, flags=re.IGNORECASE
1303 )
1304 clean_title = re.sub(
1305 r"\s*Published in.*", "", clean_title, flags=re.IGNORECASE
1306 )
1307 clean_title = re.sub(
1308 r"\s*Volume.*", "", clean_title, flags=re.IGNORECASE
1309 )
1310 clean_title = re.sub(
1311 r"\s*Pages.*", "", clean_title, flags=re.IGNORECASE
1312 )
1313 clean_title = clean_title.strip()
1315 # TY - Type of reference (ELEC for electronic source/website)
1316 lines.append("TY - ELEC")
1318 # ID - Reference ID
1319 lines.append(f"ID - ref{ref_id}")
1321 # TI - Title
1322 lines.append(f"TI - {clean_title if clean_title else title}")
1324 # AU - Authors
1325 for author in authors:
1326 lines.append(f"AU - {author}")
1328 # DO - DOI
1329 if doi:
1330 lines.append(f"DO - {doi}")
1332 # PY - Publication year (if found in title)
1333 if year:
1334 lines.append(f"PY - {year}")
1336 # UR - URL
1337 if url:
1338 lines.append(f"UR - {url}")
1340 # Try to extract domain as publisher
1341 try:
1342 from urllib.parse import urlparse
1344 parsed = urlparse(url)
1345 domain = parsed.netloc
1346 if domain.startswith("www."):
1347 domain = domain[4:]
1348 # Extract readable publisher name from domain
1349 if domain == "github.com" or domain.endswith(".github.com"):
1350 lines.append("PB - GitHub")
1351 elif domain == "arxiv.org" or domain.endswith(".arxiv.org"):
1352 lines.append("PB - arXiv")
1353 elif domain == "reddit.com" or domain.endswith(".reddit.com"):
1354 lines.append("PB - Reddit")
1355 elif (
1356 domain == "youtube.com"
1357 or domain == "m.youtube.com"
1358 or domain.endswith(".youtube.com")
1359 ):
1360 lines.append("PB - YouTube")
1361 elif domain == "medium.com" or domain.endswith(".medium.com"):
1362 lines.append("PB - Medium")
1363 elif domain == "pypi.org" or domain.endswith(".pypi.org"):
1364 lines.append("PB - Python Package Index (PyPI)")
1365 else:
1366 # Use domain as publisher
1367 lines.append(f"PB - {domain}")
1368 except (ValueError, AttributeError):
1369 pass
1371 # Y1 - Year accessed (current year)
1372 from datetime import UTC, datetime
1374 current_year = datetime.now(UTC).year
1375 lines.append(f"Y1 - {current_year}")
1377 # DA - Date accessed
1378 current_date = datetime.now(UTC).strftime("%Y/%m/%d")
1379 lines.append(f"DA - {current_date}")
1381 # LA - Language
1382 lines.append("LA - en")
1384 # ER - End of reference
1385 lines.append("ER - ")
1387 return "\n".join(lines)
1390class LaTeXExporter:
1391 """Export markdown documents to LaTeX format."""
1393 def __init__(self):
1394 # Also match Unicode lenticular brackets 【】 (U+3010 and U+3011) that LLMs sometimes generate
1395 self.citation_pattern = re.compile(r"[\[【](\d+)[\]】]")
1396 self.heading_patterns = [
1397 (re.compile(r"^# (.+)$", re.MULTILINE), r"\\section{\1}"),
1398 (re.compile(r"^## (.+)$", re.MULTILINE), r"\\subsection{\1}"),
1399 (re.compile(r"^### (.+)$", re.MULTILINE), r"\\subsubsection{\1}"),
1400 ]
1401 self.emphasis_patterns = [
1402 (re.compile(r"\*\*(.+?)\*\*"), r"\\textbf{\1}"),
1403 (re.compile(r"\*(.+?)\*"), r"\\textit{\1}"),
1404 (re.compile(r"`(.+?)`"), r"\\texttt{\1}"),
1405 ]
1407 def export_to_latex(self, content: str) -> str:
1408 """
1409 Convert markdown document to LaTeX format.
1411 Args:
1412 content: Markdown content
1414 Returns:
1415 LaTeX formatted content
1416 """
1417 latex_content = self._create_latex_header()
1419 # Convert markdown to LaTeX
1420 body_content = content
1422 # Escape special LaTeX characters but preserve math mode
1423 # Split by $ to preserve math sections
1424 parts = body_content.split("$")
1425 for i in range(len(parts)):
1426 # Even indices are outside math mode
1427 if i % 2 == 0:
1428 # Only escape if not inside $$
1429 if not (
1430 i > 0
1431 and parts[i - 1] == ""
1432 and i < len(parts) - 1
1433 and parts[i + 1] == ""
1434 ):
1435 # Preserve certain patterns that will be processed later
1436 # like headings (#), emphasis (*), and citations ([n])
1437 lines = parts[i].split("\n")
1438 for j, line in enumerate(lines):
1439 # Don't escape lines that start with # (headings)
1440 if not line.strip().startswith("#"):
1441 # Don't escape emphasis markers or citations for now
1442 # They'll be handled by their own patterns
1443 temp_line = line
1444 # Escape special chars except *, #, [, ]
1445 temp_line = temp_line.replace("&", r"\&")
1446 temp_line = temp_line.replace("%", r"\%")
1447 temp_line = temp_line.replace("_", r"\_")
1448 # Don't escape { } inside citations
1449 lines[j] = temp_line
1450 parts[i] = "\n".join(lines)
1451 body_content = "$".join(parts)
1453 # Convert headings
1454 for pattern, replacement in self.heading_patterns:
1455 body_content = pattern.sub(replacement, body_content)
1457 # Convert emphasis
1458 for pattern, replacement in self.emphasis_patterns:
1459 body_content = pattern.sub(replacement, body_content)
1461 # Convert citations to LaTeX \cite{} format
1462 body_content = self.citation_pattern.sub(r"\\cite{\1}", body_content)
1464 # Convert lists
1465 body_content = self._convert_lists(body_content)
1467 # Add body content
1468 latex_content += body_content
1470 # Add bibliography section
1471 latex_content += self._create_bibliography(content)
1473 # Add footer
1474 latex_content += self._create_latex_footer()
1476 return latex_content
1478 def _create_latex_header(self) -> str:
1479 """Create LaTeX document header."""
1480 return r"""\documentclass[12pt]{article}
1481\usepackage[utf8]{inputenc}
1482\usepackage{hyperref}
1483\usepackage{cite}
1484\usepackage{url}
1486\title{Research Report}
1487\date{\today}
1489\begin{document}
1490\maketitle
1492"""
1494 def _create_latex_footer(self) -> str:
1495 """Create LaTeX document footer."""
1496 return "\n\\end{document}\n"
1498 def _escape_latex(self, text: str) -> str:
1499 """Escape special LaTeX characters in text."""
1500 # Escape special LaTeX characters
1501 replacements = [
1502 ("\\", r"\textbackslash{}"), # Must be first
1503 ("&", r"\&"),
1504 ("%", r"\%"),
1505 ("$", r"\$"),
1506 ("#", r"\#"),
1507 ("_", r"\_"),
1508 ("{", r"\{"),
1509 ("}", r"\}"),
1510 ("~", r"\textasciitilde{}"),
1511 ("^", r"\textasciicircum{}"),
1512 ]
1514 for old, new in replacements:
1515 text = text.replace(old, new)
1517 return text
1519 def _convert_lists(self, content: str) -> str:
1520 """Convert markdown lists to LaTeX format."""
1521 # Simple conversion for bullet points
1522 content = re.sub(r"^- (.+)$", r"\\item \1", content, flags=re.MULTILINE)
1524 # Add itemize environment around list items
1525 lines = content.split("\n")
1526 result = []
1527 in_list = False
1529 for line in lines:
1530 if line.strip().startswith("\\item"):
1531 if not in_list:
1532 result.append("\\begin{itemize}")
1533 in_list = True
1534 result.append(line)
1535 else:
1536 if in_list and line.strip():
1537 result.append("\\end{itemize}")
1538 in_list = False
1539 result.append(line)
1541 if in_list:
1542 result.append("\\end{itemize}")
1544 return "\n".join(result)
1546 def _create_bibliography(self, content: str) -> str:
1547 """Extract sources and create LaTeX bibliography."""
1548 sources_start, _on_sentinel = find_sources_section(content)
1549 if sources_start == -1:
1550 return ""
1552 sources_content = content[sources_start:]
1553 entries = _collect_bibliography_entries(sources_content)
1555 bibliography = "\n\\begin{thebibliography}{99}\n"
1557 for citation_num in sorted(entries, key=_sort_key):
1558 title, url = entries[citation_num]
1559 escaped_title = self._escape_latex(title)
1561 # NOTE: thebibliography numbers entries by POSITION, so a gap
1562 # in the index sequence makes \\cite{5} print [3]. That is
1563 # pre-existing and unchanged here — an explicit
1564 # ``\\bibitem[label]{key}`` would fix it but changes an output
1565 # format existing tests pin, so it belongs in its own PR.
1566 # Sorting at least keeps the common contiguous case correct.
1567 item = f"\\bibitem{{{citation_num}}}"
1568 safe_url = _safe_bibtex_url(url)
1569 if safe_url:
1570 bibliography += f"{item} {escaped_title}. \\url{{{safe_url}}}\n"
1571 else:
1572 bibliography += f"{item} {escaped_title}.\n"
1574 bibliography += "\\end{thebibliography}\n"
1576 return bibliography