Coverage for src/local_deep_research/research_library/downloaders/arxiv.py: 97%
333 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2arXiv PDF and Text Downloader
3"""
5import re
6from dataclasses import dataclass
7from enum import StrEnum
8from typing import Callable, Dict, Final, Iterator, Optional
9from urllib.parse import urlparse
11from bs4 import BeautifulSoup, NavigableString, PageElement, Tag
12from loguru import logger
14from ...security import redact_url_for_log
16from ...constants import USER_AGENT
17from ...security.egress.policy import PolicyDeniedError
18from ...utilities.arxiv import extract_arxiv_id, is_arxiv_paper_url
19from ...utilities.arxiv_api import arxiv_api_request_gate
20from .base import ContentType, DownloadResult
21from .html import HTMLDownloader
24class ArxivTextSource(StrEnum):
25 """Canonical source used to produce arXiv text."""
27 ARXIV_HTML = "arxiv_html"
28 LOCAL_PDF = "local_pdf"
29 ARXIV_API = "arxiv_api"
32@dataclass(frozen=True, slots=True)
33class ArxivTextResult:
34 """Text paired with its exact arXiv production source."""
36 text: str
37 source: ArxivTextSource
40def attribute_arxiv_text(text: str, arxiv_id: str) -> str:
41 """Append the ``Source: https://arxiv.org/abs/<id>`` attribution line.
43 Applied to PDF and API metadata text (not to the HTML rendition), by
44 the downloader and by ``DownloadService`` for text it extracts from a
45 stored PDF, so text saved with the same provenance labels reads the
46 same whichever path produced it.
47 """
48 return f"{text.rstrip()}\n\nSource: https://arxiv.org/abs/{arxiv_id}"
51class ArxivFullTextStatus(StrEnum):
52 """How a full-text-only fetch (``download_full_text``) ended."""
54 # Full text was produced (``ArxivFullTextOutcome.result`` holds it).
55 TEXT = "text"
56 # The PDF was downloaded but extraction produced no text: an
57 # image-only scan, or the extraction ceilings hit before any text.
58 # The download and the extraction were both paid for.
59 PDF_WITHOUT_TEXT = "pdf_without_text"
60 # No full text and no PDF bytes, and arXiv answered every request:
61 # the HTML leg found no usable rendition (404/410, a non-HTML answer,
62 # or a page that is not a rendition) and the PDF request was answered
63 # 404/410 or with something that is not a PDF. The paper has no full
64 # text to fetch, so a caller budgeting fetches can refund this one.
65 NOT_FETCHED = "not_fetched"
66 # No full text and no PDF bytes because a request failed rather than
67 # being answered: a timeout, a connection error, a 5xx (other than a
68 # PDF 503) or other unexpected status, or the decoded-body cap, on
69 # either leg. Nothing says the paper lacks full text; the host is
70 # failing, so a caller fetching several papers should stop sending it
71 # requests instead of paying a timeout per paper.
72 FETCH_FAILED = "fetch_failed"
73 # As FETCH_FAILED, but arXiv answered the PDF request with HTTP 429 or
74 # 503: the host is rate limiting or unavailable, so a caller fetching
75 # several papers should stop sending it requests. The PDF leg of
76 # ``download_full_text`` makes a single attempt and does not retry
77 # into the rate limit.
78 RATE_LIMITED = "rate_limited"
81@dataclass(frozen=True, slots=True)
82class ArxivFullTextOutcome:
83 """Result of ``download_full_text``: the text, and how the fetch ended."""
85 status: ArxivFullTextStatus
86 result: ArxivTextResult | None = None
89# arXiv asks harvesters to use this host, which carries an up-to-date copy
90# of the corpus and is "specifically set aside for programmatic access"
91# (https://info.arxiv.org/help/bulk_data.html), rather than arxiv.org,
92# whose capacity is kept for interactive readers. It serves the same
93# /html/{id} and /pdf/{id} paths. Automated callers (the search engine's
94# full-text step) fetch from it; user-initiated library downloads keep
95# arxiv.org.
96ARXIV_EXPORT_HOST: Final = "export.arxiv.org"
99# HTML must be at least this fraction of the PDF text already in hand before it
100# is allowed to replace it. arXiv answers /html/{id} with a stub for papers that
101# have no HTML rendition, and such a stub clears the extraction pipeline's 50
102# character floor while carrying none of the paper.
103MIN_HTML_TO_PDF_TEXT_RATIO = 0.5
105# Absolute floor for HTML accepted with no PDF text to compare against.
106# The ratio above only protects callers that already hold PDF bytes; the
107# text-first paths (ContentFetcher.fetch, pipeline.fetch_and_extract, and
108# download_service's first-ever text extraction) call in with none, and a
109# stub clears the extraction pipeline's 50 character floor easily. The
110# known stub extracts to ~260 characters and real LaTeXML renditions run to
111# tens of thousands, so this sits about 8x above the former and an order of
112# magnitude below the latter. It is a heuristic -- arXiv does not mark stub
113# pages distinctly -- but failing it is cheap: download_text_with_source
114# then falls through to downloading the PDF, which is exactly what these
115# paths did before HTML-first existed.
116MIN_STANDALONE_HTML_TEXT_LENGTH = 2000
118# Upper bound on the number of <math> elements a rendition may carry before
119# the TeX rewrite is abandoned in favour of the PDF. The rewrite itself is
120# linear, nested <math> included (see _rewrite_math_to_tex, which rewrites
121# only outermost elements), so this is not a complexity guard: it
122# caps the work the rewrite can be made to do, and nothing more. It does not
123# bound the BeautifulSoup parse that precedes it, whose cost scales with the
124# page's total element count rather than its equation count and which every
125# HTML download has always paid; the only limits there are the HTTP timeout
126# and SafeSession's 1 GiB body cap. Real arXiv papers sit orders of magnitude
127# below the bound -- a heavy mathematics paper runs to a few hundred
128# equations -- so exceeding it marks a page the PDF serves better.
129MAX_MATH_ELEMENTS = 5000
131# MathML reaches these pages namespace-prefixed as often as not, and lxml's
132# HTML parser keeps the prefix in the tag name (``<m:math>`` parses to a tag
133# named "m:math"). A literal "math" match therefore finds nothing on such a
134# page and the rendition is accepted with its MathML left in place. What
135# the extractors then make of the raw MathML depends on the page rather than
136# on which extractor wins the tiebreak: either one may drop the element, so
137# the equation is missing from the stored text (with no PDF fallback once
138# the text clears the standalone floor), or emit its text nodes glued to the
139# TeX ("x\\frac{1}{2}" for an equation whose visible symbol is "x").
140# Matching on the local name rewrites both spellings, so the stored text is
141# the clean TeX either way.
142_MATHML_LOCAL_NAME: Final = re.compile(r"(?:^|:)math$", re.IGNORECASE)
143_TEX_ANNOTATION_LOCAL_NAME: Final = re.compile(
144 r"(?:^|:)annotation$", re.IGNORECASE
145)
146# The visible-text tier of the ladder below must not read annotation
147# markup: <annotation> children of other encodings (application/x-asy, ...)
148# and <annotation-xml> (Content MathML) carry hidden machine-readable
149# state, not rendered characters. Anchored and suffixed so a literal
150# "annotation" matches, "annotation-xml" matches, and nothing longer does.
151_HIDDEN_ANNOTATION_LOCAL_NAME: Final = re.compile(
152 r"(?:^|:)annotation(?:-xml)?$", re.IGNORECASE
153)
156class ArxivDownloader(HTMLDownloader):
157 """Download arXiv PDFs and produce text from canonical arXiv sources."""
159 # Set per instance by __init__; the class default covers instances
160 # built without it (test doubles, ``__new__``).
161 _fetch_host: str = "arxiv.org"
163 def __init__(
164 self,
165 timeout: int = 30,
166 language: str = "English",
167 *,
168 fetch_host: str = "arxiv.org",
169 ):
170 """``fetch_host`` is the host the HTML and PDF legs request from.
172 It is ``arxiv.org`` or ``ARXIV_EXPORT_HOST``; any other value is
173 rejected. Attribution (``Source: https://arxiv.org/abs/...``) and
174 the response checks are independent of it: both hosts sit inside
175 the ``.arxiv.org`` trust boundary ``_is_matching_arxiv_paper_url``
176 applies.
177 """
178 if fetch_host not in ("arxiv.org", ARXIV_EXPORT_HOST):
179 raise ValueError("fetch_host must be arxiv.org or export.arxiv.org")
180 self._fetch_host = fetch_host
181 super().__init__(timeout=timeout, language=language)
182 session = self.session
183 if session is not None: 183 ↛ exitline 183 didn't return from function '__init__' because the condition on line 183 was always true
184 session.headers.update({"User-Agent": USER_AGENT})
186 def can_handle(self, url: str) -> bool:
187 """Check if URL names a specific arXiv paper.
189 Host membership is not enough. Every entry point below needs an
190 identifier to build a canonical URL from, so a family URL without one
191 -- a listing, an author page, or the legacy ``/ftp/`` PDF layout --
192 would be claimed only to be skipped, starving the downloader that can
193 actually serve it (``DirectPDFDownloader`` for ``/ftp/`` PDFs).
194 """
195 return is_arxiv_paper_url(url)
197 def download(
198 self, url: str, content_type: ContentType = ContentType.PDF
199 ) -> Optional[bytes]:
200 """Download content from arXiv."""
201 if content_type == ContentType.TEXT:
202 text = self.download_text(url)
203 return text.encode("utf-8", errors="ignore") if text else None
204 return self._download_pdf(url)
206 def download_with_result(
207 self, url: str, content_type: ContentType = ContentType.PDF
208 ) -> DownloadResult:
209 """Download content and return detailed result with skip reason."""
210 # Extract arXiv ID
211 arxiv_id = self._extract_arxiv_id(url)
212 if not arxiv_id:
213 return DownloadResult(
214 skip_reason="Invalid arXiv URL - could not extract article ID"
215 )
217 if content_type == ContentType.TEXT:
218 result = self.download_text_with_source(url)
219 if result and result.text:
220 return DownloadResult(
221 content=result.text.encode("utf-8", errors="ignore"),
222 is_success=True,
223 content_is_complete=result.source
224 != ArxivTextSource.ARXIV_API,
225 )
226 return DownloadResult(
227 skip_reason=f"Could not retrieve full text for arXiv:{arxiv_id}"
228 )
229 # Download PDF
230 pdf_url = self._pdf_url(arxiv_id)
231 logger.info(f"Downloading arXiv PDF: {arxiv_id}")
233 pdf_content = super()._download_pdf(pdf_url)
234 if pdf_content:
235 return DownloadResult(content=pdf_content, is_success=True)
236 return DownloadResult(
237 skip_reason=f"Failed to download PDF for arXiv:{arxiv_id} - server may be unavailable"
238 )
240 def _download_pdf(
241 self,
242 url: str,
243 headers: Optional[Dict[str, str]] = None,
244 *,
245 max_attempts: int = 3,
246 on_rate_limited: Optional[Callable[[], None]] = None,
247 on_transport_failure: Optional[Callable[[], None]] = None,
248 ) -> Optional[bytes]:
249 """Download PDF from arXiv.
251 ``max_attempts``, ``on_rate_limited`` and ``on_transport_failure``
252 are passed through to ``BaseDownloader._download_pdf``.
253 """
254 # Extract arXiv ID
255 arxiv_id = self._extract_arxiv_id(url)
256 if not arxiv_id:
257 logger.error(
258 f"Could not extract arXiv ID from {redact_url_for_log(url)}"
259 )
260 return None
262 # Construct PDF URL
263 pdf_url = self._pdf_url(arxiv_id)
265 logger.info(f"Downloading arXiv PDF: {arxiv_id}")
267 # Use honest user agent - arXiv supports academic tools with proper identification
268 enhanced_headers = {
269 "User-Agent": USER_AGENT,
270 "Accept": "application/pdf,application/octet-stream,*/*",
271 "Accept-Language": "en-US,en;q=0.9",
272 "Accept-Encoding": "gzip, deflate, br",
273 "Connection": "keep-alive",
274 }
276 return super()._download_pdf(
277 pdf_url,
278 headers=enhanced_headers,
279 max_attempts=max_attempts,
280 on_rate_limited=on_rate_limited,
281 on_transport_failure=on_transport_failure,
282 )
284 def download_text(
285 self, url: str, pdf_content: bytes | None = None
286 ) -> str | None:
287 """Return official HTML, PDF text, or API metadata for an arXiv URL.
289 Supplied PDF bytes are reused even when empty. When they are omitted,
290 the canonical arXiv PDF is downloaded once. ar5iv URLs are accepted
291 only for identifier normalization and are never fetched directly.
292 """
293 result = self.download_text_with_source(url, pdf_content=pdf_content)
294 return result.text if result is not None else None
296 def _pdf_url(self, arxiv_id: str) -> str:
297 """The PDF URL on ``fetch_host``.
299 ``export.arxiv.org`` answers ``/pdf/{id}.pdf`` with a 301 to the
300 suffix-less path, so on that host the suffix is left off to avoid
301 an unnecessary redirect. (``SafeSession.resolve_redirects`` rejects
302 an oversized redirect ``Content-Length`` and discards redirect bodies
303 before Requests can buffer them.) ``arxiv.org`` keeps the URL it has
304 always been sent.
305 """
306 if self._fetch_host == ARXIV_EXPORT_HOST:
307 return f"https://{ARXIV_EXPORT_HOST}/pdf/{arxiv_id}"
308 return f"https://arxiv.org/pdf/{arxiv_id}.pdf"
310 def download_text_with_source(
311 self, url: str, pdf_content: bytes | None = None
312 ) -> ArxivTextResult | None:
313 """Return arXiv text with the exact source that produced it.
315 Transport failures on every leg are handled here, not raised: a
316 paper whose HTML and PDF legs both fail yields the API metadata
317 (``ArxivTextSource.ARXIV_API``) or ``None``.
318 """
319 arxiv_id = self._extract_arxiv_id(url)
320 if not arxiv_id:
321 return None
323 result, _ = self._html_then_pdf_text(url, arxiv_id, pdf_content)
324 if result is not None:
325 return result
327 api_text = self._fetch_from_arxiv_api(arxiv_id)
328 if api_text:
329 return ArxivTextResult(
330 attribute_arxiv_text(api_text, arxiv_id),
331 ArxivTextSource.ARXIV_API,
332 )
333 return None
335 def download_full_text(self, url: str) -> ArxivFullTextOutcome:
336 """Return full text only (HTML rendition, then PDF), and how it ended.
338 For callers that already hold the abstract: the API metadata leg is
339 never requested, because it yields only the abstract. The status
340 tells the ways a fetch can end without text apart, so a caller
341 budgeting fetches can refund only a paper that has no full text:
342 ``NOT_FETCHED`` when arXiv answered that there is none (no
343 rendition, and a PDF request answered 404/410 or with no PDF),
344 ``FETCH_FAILED`` when a request failed instead of being answered
345 (timeout, connection error, a failed DNS lookup or other refusal by
346 the URL validator, 5xx, the decoded-body cap, any other request
347 error, on either leg), ``RATE_LIMITED`` for a PDF answered 429/503, and
348 ``PDF_WITHOUT_TEXT`` for a PDF that downloaded but yielded no text
349 (the download and extraction were paid for). A URL naming no paper
350 is ``NOT_FETCHED``.
352 The PDF leg makes a single attempt: it is not retried on HTTP
353 429/503, timeouts or connection errors, so a fetch costs at most
354 two requests (HTML, then PDF). ``FETCH_FAILED`` and
355 ``RATE_LIMITED`` tell a caller fetching several papers to stop
356 instead of sending a failing host a request (and a timeout) per
357 paper. Those give-ups are logged as warnings, not errors.
358 """
359 arxiv_id = self._extract_arxiv_id(url)
360 if not arxiv_id:
361 return ArxivFullTextOutcome(ArxivFullTextStatus.NOT_FETCHED)
362 rate_limited = False
363 transport_failed = False
365 def _note_rate_limited() -> None:
366 nonlocal rate_limited
367 rate_limited = True
369 def _note_transport_failure() -> None:
370 nonlocal transport_failed
371 transport_failed = True
373 result, pdf_in_hand = self._html_then_pdf_text(
374 url,
375 arxiv_id,
376 None,
377 pdf_max_attempts=1,
378 on_pdf_rate_limited=_note_rate_limited,
379 on_transport_failure=_note_transport_failure,
380 )
381 if result is not None:
382 return ArxivFullTextOutcome(ArxivFullTextStatus.TEXT, result)
383 if pdf_in_hand:
384 return ArxivFullTextOutcome(ArxivFullTextStatus.PDF_WITHOUT_TEXT)
385 if rate_limited:
386 return ArxivFullTextOutcome(ArxivFullTextStatus.RATE_LIMITED)
387 if transport_failed:
388 return ArxivFullTextOutcome(ArxivFullTextStatus.FETCH_FAILED)
389 return ArxivFullTextOutcome(ArxivFullTextStatus.NOT_FETCHED)
391 def _html_then_pdf_text(
392 self,
393 url: str,
394 arxiv_id: str,
395 pdf_content: bytes | None,
396 *,
397 pdf_max_attempts: int | None = None,
398 on_pdf_rate_limited: Callable[[], None] | None = None,
399 on_transport_failure: Callable[[], None] | None = None,
400 ) -> tuple[ArxivTextResult | None, bool]:
401 """Produce text from the HTML rendition, then the PDF.
403 Returns the text (or ``None``) and whether PDF bytes were in hand,
404 supplied or downloaded, whether or not they yielded any text.
406 ``pdf_max_attempts`` and ``on_pdf_rate_limited`` are forwarded to
407 ``_download_pdf`` when the PDF is downloaded; left unset, the PDF
408 download keeps its default retries. ``on_transport_failure`` is
409 forwarded to the HTML fetch and, with ``pdf_max_attempts``, to the
410 PDF download.
411 """
412 # PDF bytes already in hand set the quality floor for HTML, so their
413 # text is extracted before HTML is considered.
414 pdf_text = (
415 self.extract_text_from_pdf(pdf_content)
416 if pdf_content is not None
417 else None
418 )
419 pdf_in_hand = pdf_content is not None
421 # HTML is fetched even when PDF bytes are already in hand: preferring
422 # the official HTML rendition is the point of this path, and the PDF
423 # text above is what sets the quality floor it has to clear
424 # (_html_text_beats_pdf_text). That costs one request the PDF-only
425 # path did not make; the caller's retry budget is what bounds it.
426 if on_transport_failure is None:
427 html_text = self._download_html_text(arxiv_id)
428 else:
429 html_text = self._download_html_text(
430 arxiv_id, on_transport_failure=on_transport_failure
431 )
432 if html_text and self._html_text_beats_pdf_text(
433 html_text, pdf_text, arxiv_id
434 ):
435 return (
436 ArxivTextResult(html_text, ArxivTextSource.ARXIV_HTML),
437 pdf_in_hand,
438 )
440 if pdf_content is None:
441 logger.info(f"Downloading arXiv PDF for full text: {arxiv_id}")
442 if pdf_max_attempts is None:
443 downloaded_pdf = self._download_pdf(url)
444 else:
445 downloaded_pdf = self._download_pdf(
446 url,
447 max_attempts=pdf_max_attempts,
448 on_rate_limited=on_pdf_rate_limited,
449 on_transport_failure=on_transport_failure,
450 )
451 if downloaded_pdf is not None:
452 pdf_in_hand = True
453 pdf_text = self.extract_text_from_pdf(downloaded_pdf)
455 if pdf_text:
456 return (
457 ArxivTextResult(
458 attribute_arxiv_text(pdf_text, arxiv_id),
459 ArxivTextSource.LOCAL_PDF,
460 ),
461 pdf_in_hand,
462 )
463 return None, pdf_in_hand
465 def _download_html_text(
466 self,
467 arxiv_id: str,
468 *,
469 on_transport_failure: Callable[[], None] | None = None,
470 ) -> str | None:
471 """Extract normalized text from official arXiv HTML when usable.
473 ``on_transport_failure`` is forwarded to the HTML fetch (see
474 ``HTMLDownloader._fetch_html_with_final_url``).
475 """
476 html_url = f"https://{self._fetch_host}/html/{arxiv_id}"
477 # Single-shot HTML leg: the full-text flow implements its own
478 # backoff/circuit-breaking across papers (NOT_FETCHED vs
479 # FETCH_FAILED accounting), so the generic fetch must not
480 # retry timeouts/429s underneath it and triple the per-paper
481 # cost against a rate-limiting host.
482 if on_transport_failure is None:
483 html, final_url = self._fetch_html_with_final_url(
484 html_url, max_attempts=1
485 )
486 else:
487 html, final_url = self._fetch_html_with_final_url(
488 html_url,
489 on_transport_failure=on_transport_failure,
490 max_attempts=1,
491 )
492 if not html:
493 return None
494 if not self._stayed_on_arxiv_html(final_url, arxiv_id):
495 logger.info(
496 "arXiv HTML request for {} redirected to {}; not a rendition",
497 arxiv_id,
498 redact_url_for_log(final_url),
499 )
500 return None
502 try:
503 soup = BeautifulSoup(html, "lxml")
504 if not self._rewrite_math_to_tex(soup, arxiv_id):
505 return None
507 canonical_url = f"https://arxiv.org/abs/{arxiv_id}"
508 extracted = self._extract_content(str(soup), canonical_url)
509 if not extracted:
510 return None
511 text = self._format_extracted_content(extracted)
512 return text if text.strip() else None
513 except Exception:
514 logger.warning(
515 "arXiv HTML unusable; trying fallbacks: {}", arxiv_id
516 )
517 return None
519 @staticmethod
520 def _tex_for_math_element(math: Tag) -> str:
521 """The TeX one <math> element carries, or ``""`` when it carries none.
523 Four sources, each a strictly weaker guarantee than the one before.
524 An ``<annotation encoding="application/x-tex">`` child first because
525 it is the explicit one; then an
526 ``<annotation encoding="application/x-tex+html">`` child, the same
527 TeX with the HTML LaTeXML kept around it; then the ``alttext``
528 attribute LaTeXML always writes onto the element itself; and finally
529 the element's visible MathML text, read with the annotation
530 subtrees skipped so annotation state is never mistaken for rendered
531 characters, so a bare unannotated equation
532 still yields its rendered characters instead of its annotation state
533 (#4783). Only annotations are removed: presentation elements that
534 lay out without drawing, such as ``<mphantom>``, keep their text in
535 this tier. An empty source falls through rather than abandoning the
536 page -- an element can carry an empty annotation and a usable one
537 beside it, or an empty annotation and a usable attribute, and there
538 is no reason to prefer the empty one. Every annotation of each
539 encoding is examined for that reason, not just the first descendant
540 -- but only the outermost of a nested run: an annotation nested in a
541 same-encoding annotation contributes a subset of its ancestor's
542 text, so once the ancestor reads as empty every annotation inside it
543 does too. Reading only outermost matches keeps their subtrees
544 disjoint, so each encoding's search, text extraction included, is
545 one linear pass over the element; re-reading every level of a
546 nested chain would be quadratic in its depth, and this HTML is
547 author-controlled.
549 ``<annotation-xml>`` is deliberately not a source: it holds a
550 different encoding, the anchored matcher above already excludes it,
551 and the visible-text tier skips it — along with every other
552 non-TeX ``<annotation>`` — in the text it reads, so its hidden
553 text can neither supply an empty equation nor pollute a rendered
554 one. A node whose only text is annotation text reads as empty and
555 takes the PDF exit.
556 """
557 for encoding in ("application/x-tex", "application/x-tex+html"):
558 for annotation in ArxivDownloader._outermost_tex_annotations(
559 math, encoding
560 ):
561 tex = annotation.get_text().strip()
562 if tex:
563 return tex
564 alttext = math.get("alttext")
565 if isinstance(alttext, str) and alttext.strip():
566 return alttext.strip()
567 return ArxivDownloader._visible_math_text(math)
569 @staticmethod
570 def _outermost_tex_annotations(math: Tag, encoding: str) -> Iterator[Tag]:
571 """``<annotation encoding=...>`` descendants with no such ancestor.
573 Yielded lazily in document order. One iterative walk that does not
574 descend into a match, so every node is visited at most once and the
575 yielded subtrees are disjoint -- reading each one's text is linear
576 in the element's size in total, however deeply matches nest.
577 """
578 stack: list[Tag] = [
579 child for child in reversed(math.contents) if isinstance(child, Tag)
580 ]
581 while stack:
582 node = stack.pop()
583 if (
584 _TEX_ANNOTATION_LOCAL_NAME.search(node.name or "")
585 and node.get("encoding") == encoding
586 ):
587 yield node
588 continue
589 stack.extend(
590 child
591 for child in reversed(node.contents)
592 if isinstance(child, Tag)
593 )
595 @staticmethod
596 def _visible_math_text(math: Tag) -> str:
597 """``math.get_text(" ", strip=True)`` minus annotation subtrees.
599 Hidden machine-readable state -- Content MathML inside
600 ``<annotation-xml>``, other encodings inside ``<annotation>`` -- is
601 never emitted as the equation's rendered characters. One iterative
602 walk that skips those subtrees, so each node is visited once and the
603 tree is left untouched. (A ``copy.deepcopy`` of the element with the
604 annotations decomposed gives the same text, but bs4's deepcopy
605 re-walks the copy on every insert, which is quadratic in the depth
606 of nested markup.) Strings are filtered by the same types
607 ``get_text`` keeps, so comments and the like stay out.
608 """
609 types = math.interesting_string_types or Tag.MAIN_CONTENT_STRING_TYPES
610 parts: list[str] = []
611 stack: list[PageElement] = list(reversed(math.contents))
612 while stack:
613 node = stack.pop()
614 if isinstance(node, Tag):
615 if _HIDDEN_ANNOTATION_LOCAL_NAME.search(node.name or ""):
616 continue
617 stack.extend(reversed(node.contents))
618 elif isinstance(node, NavigableString) and type(node) in types:
619 stripped = node.strip()
620 if stripped: 620 ↛ 612line 620 didn't jump to line 612 because the condition on line 620 was always true
621 parts.append(stripped)
622 return " ".join(parts)
624 @staticmethod
625 def _rewrite_math_to_tex(soup: BeautifulSoup, arxiv_id: str) -> bool:
626 """Replace every <math> element with its TeX annotation, in place.
628 Returns False when the rendition must be abandoned for the PDF. Two
629 distinct cases do that, each logged at debug so they are told apart in
630 a report: the page carries more than MAX_MATH_ELEMENTS equations, or a
631 <math> offers no TeX at all.
633 "No TeX at all" is narrower than it used to be. The annotation child
634 is not the only place LaTeXML writes the TeX -- it always writes it
635 into the ``alttext`` attribute as well, while the parallel
636 <annotation> markup depends on how arXiv invokes it, so a rendition
637 carrying only ``alttext`` used to take the PDF exit with its TeX
638 sitting right there on the element (#6414). Both sources are read
639 now, annotation first. Partially annotated renditions go further and
640 mix TeX-annotated equations with bare MathML (#4783): the recovery
641 ladder reads x-tex annotations, then x-tex+html ones, then
642 ``alttext``, then the element's visible MathML text, so a node kills
643 the page only when it is genuinely empty -- no annotation, no
644 attribute, and no rendered characters to keep.
646 Each element is rewritten in place -- renamed to an inline <span>
647 holding the TeX -- rather than with ``Tag.replace_with``. replace_with
648 locates the element with a linear scan of its parent's contents, so
649 rewriting n equations that share one parent costs O(n^2); arXiv
650 renders this HTML from author-submitted LaTeX, and it is reached on
651 the search path. Rewriting in place is linear and keeps bs4's own
652 escaping on serialization.
654 Only outermost <math> elements are rewritten. Each one's TeX comes
655 from its own subtree (the annotation search, and the visible-text
656 tier's walk), so a <math> nested inside another is covered by its
657 ancestor and is discarded with the ancestor's children. Visiting
658 the nested ones as well would re-read every level of a nested
659 chain, making the rewrite quadratic in the nesting depth (nested
660 <math> is not valid MathML, but this HTML is author-controlled).
661 Outermost elements have disjoint subtrees, so the total work is
662 linear in the page size.
663 """
664 math_elements = soup.find_all(_MATHML_LOCAL_NAME)
665 if len(math_elements) > MAX_MATH_ELEMENTS:
666 logger.debug(
667 "arXiv HTML for {} carries {} <math> elements, above the "
668 "{} element bound; falling back to the PDF",
669 arxiv_id,
670 len(math_elements),
671 MAX_MATH_ELEMENTS,
672 )
673 return False
675 for math in ArxivDownloader._outermost_math_elements(soup):
676 tex = ArxivDownloader._tex_for_math_element(math)
677 if not tex:
678 logger.debug(
679 "arXiv HTML for {} has a <math> element carrying no "
680 "annotation of either TeX encoding, no alttext "
681 "attribute, and no visible MathML text; falling back "
682 "to the PDF",
683 arxiv_id,
684 )
685 return False
686 math.name = "span"
687 math.attrs.clear()
688 math.clear()
689 math.append(NavigableString(tex))
690 return True
692 @staticmethod
693 def _outermost_math_elements(soup: BeautifulSoup) -> list[Tag]:
694 """<math> elements with no <math> ancestor, in document order.
696 One iterative walk that does not descend into a matched element,
697 so every node is visited at most once.
698 """
699 outermost: list[Tag] = []
700 # Children are pushed in reverse so they pop in document order.
701 stack: list[Tag] = [
702 child for child in reversed(soup.contents) if isinstance(child, Tag)
703 ]
704 while stack:
705 node = stack.pop()
706 if _MATHML_LOCAL_NAME.search(node.name or ""):
707 outermost.append(node)
708 continue
709 stack.extend(
710 child
711 for child in reversed(node.contents)
712 if isinstance(child, Tag)
713 )
714 return outermost
716 @staticmethod
717 def _is_matching_arxiv_paper_url(
718 response_url: str | None, arxiv_id: str
719 ) -> bool:
720 """Require an official URL for the requested paper and version.
722 An unversioned request may resolve to a versioned rendition of the
723 same paper. An explicit version must remain exact. The host test is
724 the ``.arxiv.org`` trust boundary, so ``ar5iv.labs.arxiv.org`` -- the
725 host the real ar5iv service runs on -- is inside it and accepted,
726 while the bare ``ar5iv.org`` domain is not: it is an input identifier
727 only and is never treated as an authoritative response. That is also
728 why this does not reuse ``is_arxiv_family_url``, whose family takes
729 in ``ar5iv.org``: widening the boundary to match it would accept a
730 response from a host arXiv does not serve.
731 """
732 if not response_url: 732 ↛ 733line 732 didn't jump to line 733 because the condition on line 732 was never true
733 return False
734 try:
735 parsed = urlparse(response_url)
736 hostname = (parsed.hostname or "").lower().rstrip(".")
737 expected_port = {"https": 443, "http": 80}.get(parsed.scheme)
738 if (
739 expected_port is None
740 or parsed.port not in {None, expected_port}
741 or parsed.username is not None
742 or parsed.password is not None
743 or not (
744 hostname == "arxiv.org" or hostname.endswith(".arxiv.org")
745 )
746 ):
747 return False
748 except ValueError:
749 return False
750 actual_id = extract_arxiv_id(response_url)
751 if actual_id is None:
752 return False
753 if re.search(r"v\d+$", arxiv_id, re.IGNORECASE):
754 return actual_id.casefold() == arxiv_id.casefold()
755 return (
756 re.sub(r"v\d+$", "", actual_id, flags=re.IGNORECASE).casefold()
757 == arxiv_id.casefold()
758 )
760 @classmethod
761 def _stayed_on_arxiv_html(
762 cls, final_url: str | None, arxiv_id: str
763 ) -> bool:
764 """Accept HTTPS HTML only for the requested arXiv paper/version."""
765 if not final_url or not cls._is_matching_arxiv_paper_url(
766 final_url, arxiv_id
767 ):
768 return False
769 parsed = urlparse(final_url)
770 return parsed.scheme == "https" and parsed.path.startswith("/html/")
772 def _html_text_beats_pdf_text(
773 self, html_text: str, pdf_text: str | None, arxiv_id: str
774 ) -> bool:
775 """Report whether HTML text may replace PDF text already extracted.
777 With no PDF text in hand there is nothing to compare against, so the
778 ratio cannot help; an absolute floor stands in for it. Returning True
779 unconditionally here is what let a stub win on every text-first path.
780 """
781 if not pdf_text:
782 if len(html_text) >= MIN_STANDALONE_HTML_TEXT_LENGTH:
783 return True
784 logger.warning(
785 "arXiv HTML for {} yielded only {} characters and no PDF text "
786 "is available to compare against; below the {} character floor "
787 "for standalone HTML, so falling back to the PDF",
788 arxiv_id,
789 len(html_text),
790 MIN_STANDALONE_HTML_TEXT_LENGTH,
791 )
792 return False
793 if len(html_text) >= MIN_HTML_TO_PDF_TEXT_RATIO * len(pdf_text): 793 ↛ 794line 793 didn't jump to line 794 because the condition on line 793 was never true
794 return True
795 logger.warning(
796 "arXiv HTML for {} yielded only {} characters against {} "
797 "characters of PDF text; keeping the PDF text",
798 arxiv_id,
799 len(html_text),
800 len(pdf_text),
801 )
802 return False
804 def _extract_arxiv_id(self, url: str) -> Optional[str]:
805 """Extract arXiv ID from URL."""
806 return extract_arxiv_id(url)
808 def get_metadata(self, url: str) -> dict[str, str]:
809 """Return canonical arXiv attribution without making a request."""
810 arxiv_id = self._extract_arxiv_id(url)
811 if arxiv_id is None:
812 return {}
813 return {"url": f"https://arxiv.org/abs/{arxiv_id}"}
815 def _fetch_from_arxiv_api(self, arxiv_id: str) -> Optional[str]:
816 """Fetch abstract and metadata from arXiv API."""
817 engine_type = "arxiv_api"
818 session = self.session
819 if session is None: 819 ↛ 820line 819 didn't jump to line 820 because the condition on line 819 was never true
820 self.rate_tracker.record_outcome(
821 engine_type=engine_type,
822 wait_time=0.0,
823 success=False,
824 retry_count=1,
825 error_type="SessionUnavailable",
826 )
827 return None
829 adaptive_wait = self.rate_tracker.apply_rate_limit(engine_type)
830 api_text: str | None = None
831 error_type: str | None = None
832 try:
833 with arxiv_api_request_gate():
834 response = session.get(
835 "https://export.arxiv.org/api/query",
836 params={"id_list": arxiv_id},
837 timeout=10,
838 allow_redirects=False,
839 )
841 if response.status_code != 200:
842 error_type = f"HTTP_{response.status_code}"
843 else:
844 # Parse the Atom feed response
845 # Use defusedxml to prevent XXE attacks
846 from defusedxml import ElementTree as ET
848 root = ET.fromstring(response.text)
850 # Define namespaces (URIs are identifiers, not URLs to fetch)
851 ns = {
852 "atom": "http://www.w3.org/2005/Atom", # DevSkim: ignore DS137138
853 "arxiv": "http://arxiv.org/schemas/atom", # DevSkim: ignore DS137138
854 }
856 # Find the entry
857 entry = root.find("atom:entry", ns)
858 if entry is not None:
859 entry_id = entry.find("atom:id", ns)
860 response_url = (
861 entry_id.text.strip()
862 if entry_id is not None and entry_id.text
863 else None
864 )
865 if (
866 response_url is None
867 or not self._is_matching_arxiv_paper_url(
868 response_url, arxiv_id
869 )
870 or not urlparse(response_url).path.startswith("/abs/")
871 ):
872 entry = None
873 if entry is not None:
874 # Extract text content
875 text_parts: list[str] = []
877 # Title
878 title = entry.find("atom:title", ns)
879 if title is not None and title.text:
880 title_text = title.text.strip()
881 if title_text:
882 text_parts.append(f"Title: {title_text}")
884 # Authors
885 authors = entry.findall("atom:author", ns)
886 if authors:
887 author_names: list[str] = []
888 for author in authors:
889 name = author.find("atom:name", ns)
890 if name is not None and name.text: 890 ↛ 888line 890 didn't jump to line 888 because the condition on line 890 was always true
891 author_name = name.text.strip()
892 if author_name:
893 author_names.append(author_name)
894 if author_names:
895 text_parts.append(
896 f"Authors: {', '.join(author_names)}"
897 )
899 # Abstract
900 summary = entry.find("atom:summary", ns)
901 if summary is not None and summary.text:
902 summary_text = summary.text.strip()
903 if summary_text:
904 text_parts.append(f"\nAbstract:\n{summary_text}")
906 # Categories
907 categories = entry.findall("atom:category", ns)
908 if categories:
909 cat_terms: list[str] = []
910 for category in categories:
911 term = category.get("term")
912 if term and term.strip():
913 cat_terms.append(term.strip())
914 if cat_terms:
915 text_parts.append(
916 f"\nCategories: {', '.join(cat_terms)}"
917 )
919 if text_parts:
920 api_text = "\n".join(text_parts)
922 if api_text is None:
923 error_type = "UnusableFeed"
925 except PolicyDeniedError:
926 raise
927 except Exception as e:
928 logger.debug(f"Failed to fetch from arXiv API ({type(e).__name__})")
929 error_type = type(e).__name__
931 if api_text is not None:
932 self.rate_tracker.record_outcome(
933 engine_type=engine_type,
934 wait_time=adaptive_wait,
935 success=True,
936 retry_count=1,
937 search_result_count=1,
938 )
939 logger.info(f"Retrieved text content from arXiv API for {arxiv_id}")
940 return api_text
942 self.rate_tracker.record_outcome(
943 engine_type=engine_type,
944 wait_time=adaptive_wait,
945 success=False,
946 retry_count=1,
947 error_type=error_type,
948 )
949 return None