Coverage for src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py: 97%
230 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1import os
2import re
3import uuid
4from pathlib import Path
5from typing import Any, Dict, List, Optional
7import arxiv
8from langchain_core.language_models import BaseLLM
9from requests.exceptions import RequestException
11from ...config.paths import get_data_directory
12from ...constants import SNIPPET_LENGTH_SHORT
13from ...security import SafeSession
14from ...security.directory_creation import (
15 DirectoryCreationSecurityError,
16 create_directory,
17)
18from ...security.safe_requests import DEFAULT_TIMEOUT, MAX_RESPONSE_SIZE
19from ...security.secure_logging import logger
20from ..rate_limiting import RateLimitError
21from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
23# Canonical arXiv identifier shapes, anchored so nothing else slips through
24# before we build an egress URL from the value:
25# - new style: 2301.12345 / 2301.12345v2 (4-or-5 digit sequence)
26# - old style: math.GT/0309136 or cond-mat/0501234 (with optional version)
27_ARXIV_ID_RE = re.compile(
28 r"^(?:\d{4}\.\d{4,5}(?:v\d+)?|[a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?)$"
29)
32class ArXivSearchEngine(BaseSearchEngine):
33 """arXiv search engine implementation with two-phase approach"""
35 # Mark as public search engine
36 is_public = True
37 egress_sensitivity = Sensitivity.NON_SENSITIVE
38 egress_exposure = Exposure.EXPOSING
39 # Not a generic search engine (specialized for academic papers)
40 is_generic = False
41 # Scientific/academic search engine
42 is_scientific = True
43 is_lexical = True
44 needs_llm_relevance_filter = True
46 def __init__(
47 self,
48 max_results: int = 10,
49 sort_by: str = "relevance",
50 sort_order: str = "descending",
51 include_full_text: bool = False,
52 download_dir: Optional[str] = None,
53 max_full_text: int = 1,
54 llm: Optional[BaseLLM] = None,
55 max_filtered_results: Optional[int] = None,
56 settings_snapshot: Optional[Dict[str, Any]] = None,
57 ): # Added this parameter
58 """
59 Initialize the arXiv search engine.
61 Args:
62 max_results: Maximum number of search results
63 sort_by: Sorting criteria ('relevance', 'lastUpdatedDate', or 'submittedDate')
64 sort_order: Sort order ('ascending' or 'descending')
65 include_full_text: Whether to include full paper content in results (downloads PDF)
66 download_dir: Directory to download PDFs to (if include_full_text is True)
67 max_full_text: Maximum number of PDFs to download and process (default: 1)
68 llm: Language model for relevance filtering
69 max_filtered_results: Maximum number of results to keep after filtering
70 settings_snapshot: Settings snapshot for thread context
71 """
72 # Initialize the journal reputation filter if needed.
73 # Runs as a preview filter (before LLM relevance) because Tiers 1-3
74 # are instant data lookups — no point sending irrelevant journals
75 # through the expensive LLM relevance filter.
76 preview_filters = []
77 journal_filter = self._create_journal_filter(
78 "arxiv", llm, settings_snapshot
79 )
80 if journal_filter is not None:
81 preview_filters.append(journal_filter)
83 super().__init__(
84 llm=llm,
85 max_filtered_results=max_filtered_results,
86 max_results=max_results,
87 preview_filters=preview_filters, # type: ignore[arg-type]
88 settings_snapshot=settings_snapshot,
89 )
90 self.max_results = max(self.max_results, 25)
91 self.sort_by = sort_by
92 self.sort_order = sort_order
93 self.include_full_text = include_full_text
94 self.download_dir = download_dir
95 self.max_full_text = max_full_text
97 # Map sort parameters to arxiv package parameters
98 self.sort_criteria = {
99 "relevance": arxiv.SortCriterion.Relevance,
100 "lastUpdatedDate": arxiv.SortCriterion.LastUpdatedDate,
101 "submittedDate": arxiv.SortCriterion.SubmittedDate,
102 }
104 self.sort_directions = {
105 "ascending": arxiv.SortOrder.Ascending,
106 "descending": arxiv.SortOrder.Descending,
107 }
109 def _get_search_results(self, query: str) -> List[Any]:
110 """
111 Helper method to get search results from arXiv API.
113 Args:
114 query: The search query
116 Returns:
117 List of arXiv paper objects
118 """
119 # Configure the search client
120 sort_criteria = self.sort_criteria.get(
121 self.sort_by, arxiv.SortCriterion.Relevance
122 )
123 sort_order = self.sort_directions.get(
124 self.sort_order, arxiv.SortOrder.Descending
125 )
127 # Create the search client
128 client = arxiv.Client(page_size=self.max_results)
130 # Create the search query
131 search = arxiv.Search(
132 query=query,
133 max_results=self.max_results,
134 sort_by=sort_criteria,
135 sort_order=sort_order,
136 )
138 # Apply rate limiting before making the request
139 self._last_wait_time = self.rate_tracker.apply_rate_limit(
140 self.engine_type
141 )
143 # Get the search results
144 return list(client.results(search))
146 @staticmethod
147 def _validated_arxiv_id(paper: Any) -> Optional[str]:
148 """Return a canonical, validated arXiv id for ``paper`` or ``None``.
150 The id is derived from the arxiv-provided ``entry_id`` (e.g.
151 ``http://arxiv.org/abs/2301.12345v1``) and matched against
152 ``_ARXIV_ID_RE`` before it is ever used to build an egress URL,
153 so a malformed or unexpected value cannot be interpolated into a
154 request target.
155 """
156 entry_id = getattr(paper, "entry_id", "") or ""
157 match = re.search(r"arxiv\.org/abs/(.+)$", entry_id)
158 candidate = (match.group(1) if match else entry_id).strip()
159 if _ARXIV_ID_RE.match(candidate):
160 return candidate
161 return None
163 @staticmethod
164 def _enforce_response_size_cap(written: int) -> None:
165 """Raise if a streamed PDF body has grown past ``MAX_RESPONSE_SIZE``.
167 Split out from ``_download_pdf_safely``'s streaming loop so the
168 ``raise`` isn't directly inside that loop's own cleanup
169 ``try``/``except`` block.
170 """
171 if written > MAX_RESPONSE_SIZE:
172 raise ValueError(
173 f"arXiv PDF body exceeded {MAX_RESPONSE_SIZE} bytes "
174 "while streaming"
175 )
177 def _download_pdf_safely(self, paper: Any, dirpath: str) -> str:
178 """Download a paper PDF through the SSRF-validated ``SafeSession``.
180 Invariant: every arXiv PDF fetch made *by this engine*
181 (``ArXivSearchEngine``) goes through this gate. ``SafeSession``
182 applies SSRF pre-validation and DNS pinning to this fetch. This
183 replaces ``arxiv.Result.download_pdf``, which fetches via
184 ``urllib.request.urlretrieve`` and therefore bypasses all of those
185 controls. This is narrower than "the same session as the rest of
186 the arXiv integration": the metadata queries this engine makes
187 elsewhere (``arxiv.Client()`` in ``_get_search_results`` and
188 ``get_paper_details``) still go through the ``arxiv`` package's own
189 bare ``requests.Session()`` — unvalidated and untimed. Gating those
190 too is out of scope here.
192 It is also narrower than "every arXiv PDF fetch in this codebase":
193 ``research_library/downloaders/arxiv.py``'s ``ArxivDownloader``
194 (via ``BaseDownloader._download_pdf``) independently fetches
195 ``https://arxiv.org/pdf/{id}.pdf`` — the public host, with the
196 ``.pdf`` suffix this module omits specifically to dodge the
197 redirect-drain gap described below. It does use ``SafeSession``,
198 but sends ``Accept-Encoding: gzip, deflate, br`` and reads
199 ``response.content`` with no ``stream=True``, so it has neither
200 this call site's running-byte-count cap nor its identity-encoding
201 best-effort measure. That is a separate, untouched call site —
202 fixing it is out of scope here.
204 The filename changes from the old path's ``<id>.<sanitized
205 title>.pdf`` (``arxiv.Result._get_default_filename``) to
206 ``<id>.pdf`` here — a behavior change worth knowing about even
207 though nothing in this codebase parses the filename back apart.
209 No ``.pdf`` suffix is appended to the request URL: arxiv 2.4.1's
210 own Atom ``pdf_url`` (what ``arxiv.Result.download_pdf`` fetches)
211 has none either, e.g. ``"http://arxiv.org/pdf/2101.00001v1"``.
212 Appending ``.pdf`` makes ``export.arxiv.org`` answer with a 301 to
213 the suffix-less path, and that redirect response's own body is
214 drained by ``requests.Session.send()``'s internal
215 ``resolve_redirects()`` (``resp.content # Consume socket``)
216 *before* it is ever handed to ``SafeSession.send()`` for a size
217 check — so an oversized redirect body would be read into memory
218 unchecked. Omitting the suffix removes that hop entirely, which is
219 the minimal fix; it is not a fix to ``SafeSession`` itself (every
220 redirect hop has this gap, not just this call site). This
221 redirect-drain gap in ``SafeSession.send()`` has no tracking issue
222 filed for it yet — it is untracked, and distinct from the
223 chunked-body gap discussed below (which does have one, #6180).
225 This call passes ``stream=True`` and writes the body to
226 ``target_path`` in bounded chunks via ``iter_content`` rather than
227 reading ``response.content`` — unlike what an earlier version of
228 this comment claimed, ``stream=True`` alone does not bound memory;
229 `research_library/downloaders/generic.py`'s ``stream=True`` usage
230 is not a precedent for this pattern, since it never reads the body
231 at all (a status-code-only diagnostic probe). The running byte
232 count enforced while streaming is also the actual guard against a
233 gzip/deflate decompression bomb here: for a valid, under-cap
234 ``Content-Length``, ``SafeSession._check_response_size`` installs
235 no body guard at all (see ``safe_requests.py``), and ``requests``
236 (unlike the old ``urllib``-based path) sends
237 ``Accept-Encoding: gzip`` by default, so a small compressed
238 response could otherwise decompress far past any header-level
239 check. This fetch instead sends ``Accept-Encoding: identity`` —
240 matching what the old ``urlretrieve``-based path effectively did
241 (``urllib`` does not request compression) — as a best-effort
242 signal, since a PDF is already-compressed binary and gains
243 nothing from gzip transfer. This does not remove the
244 decompression-bomb vector outright: ``urllib3`` decodes based on
245 the *response*'s ``Content-Encoding`` header, not the request's
246 ``Accept-Encoding`` one, so a server that ignores the request
247 header and gzips the body anyway would still be transparently
248 decoded by ``iter_content`` here. What actually bounds a
249 decompression bomb, with or without gzip, is the running byte
250 count enforced while streaming above — it is computed over the
251 already-decoded chunks ``iter_content`` yields, so it caps the
252 decompressed size regardless of what encoding the origin used.
254 Connection release: if ``SafeSession``'s response-size cap rejects
255 an over-limit ``Content-Length`` response, that ``ValueError`` is
256 raised *inside* ``SafeSession.send()`` — i.e. inside the
257 ``session.get(...)`` call itself, before the ``with ... as
258 response:`` statement below ever binds. So on that path the
259 connection is released by ``_check_response_size`` calling
260 ``response.close()`` internally, not by this function's context
261 manager; the context manager only covers releasing the connection
262 for responses that *do* make it past that check (including this
263 function's own errors raised while streaming the body).
265 It does not fully fix the ``Content-Length``-absent case. The cap
266 installs a guard on ``response.raw.read()``
267 (``_install_body_guard``), and that guard does run correctly — but
268 only when the connection is delimited by the socket closing (no
269 ``Content-Length``, no ``Transfer-Encoding``). When the origin uses
270 ``Transfer-Encoding: chunked`` instead — the common case for a CDN
271 serving a PDF of unknown length — ``urllib3``'s
272 ``HTTPResponse.stream()`` (what ``requests`` uses under
273 ``iter_content``/``.content``) reads such bodies through
274 ``read_chunked()``, which pulls bytes directly off the socket via
275 ``self._fp._safe_read()`` and never calls the patched
276 ``read()`` — verified by reading the installed ``urllib3``
277 (2.7.0) source and confirming with a live chunked-response
278 server: the patched ``read()`` recorded zero calls while a full
279 chunked body was consumed. So for a genuinely chunked response,
280 ``_check_response_size`` still cannot bound memory on its own here
281 — this function's own running byte count during streaming is what
282 does that instead.
284 The ``_check_response_size`` chunked-body gap is a defect in
285 ``safe_requests.py`` itself (``_check_response_size``/
286 ``_install_body_guard``) — every ``SafeSession`` caller that
287 already uses ``stream=True`` has the same exposure for chunked
288 responses. It is independent of whether this call streams and is
289 not the same issue as #6172, which is about callers that never
290 set ``stream=True`` at all (this one now does). It has its own
291 tracking issue, #6180, and needs its own follow-up rather than
292 being fixed here.
293 """
294 arxiv_id = self._validated_arxiv_id(paper)
295 if not arxiv_id:
296 raise ValueError(
297 "Could not derive a valid arXiv id for PDF download"
298 )
300 # Build the download URL from the validated id rather than trusting
301 # an arbitrary attribute value; SafeSession re-validates it anyway.
302 # No ".pdf" suffix -- see the docstring note above for why.
303 # Use export.arxiv.org (not the public arxiv.org CDN-fronted host)
304 # to match arxiv.Result.download_pdf's default `download_domain`
305 # (see the installed `arxiv` package, ~v2.4) and the host
306 # `arxiv.Client`/`arxiv.Search` already use for the metadata API
307 # (`query_url_format = "https://export.arxiv.org/api/query?..."`).
308 # export.arxiv.org is arXiv's designated host for automated/scripted
309 # access; the public arxiv.org host is rate-limited and
310 # bot-challenged for that traffic.
311 pdf_url = f"https://export.arxiv.org/pdf/{arxiv_id}"
313 directory = Path(dirpath)
314 # `dirpath` (== self.download_dir) is reachable from per-user web
315 # settings ("search." is an allowed settings prefix; the factory
316 # splats `search.engine.web.arxiv.default_params.download_dir`
317 # straight into this engine's constructor kwargs) as well as
318 # `LDR_SEARCH_ENGINE_WEB_ARXIV_DEFAULT_PARAMS_DOWNLOAD_DIR`, so it
319 # is untrusted/user-influenced input, not a trusted constant --
320 # containment is required per directory_creation.py's own
321 # opt-in-root guidance. Contain it to a dedicated subtree of the
322 # LDR-managed data directory rather than the data directory
323 # itself: the data directory root also holds `.secret_key`
324 # (fastapi_app.py), `encrypted_databases/` (per-user encrypted DB
325 # files + `.salt`), and backups, and `create_directory` is called
326 # with `parents=True, exist_ok=True` -- with the bare data
327 # directory as root, an authenticated user could `mkdir` any
328 # not-yet-existing path under it, including pre-occupying the
329 # (computable) encrypted-DB path of a username that has not
330 # registered yet. Scoping the root to a download-only subtree
331 # keeps the blast radius of that primitive to the download area.
332 arxiv_downloads_root = get_data_directory() / "arxiv_downloads"
333 create_directory(
334 directory,
335 context="arXiv PDF download directory",
336 root=arxiv_downloads_root,
337 )
338 target_path = directory / f"{arxiv_id.replace('/', '_')}.pdf"
340 with SafeSession() as session:
341 # stream=True is intentional here — DO NOT remove it: it lets
342 # the body be written to disk in bounded chunks below instead
343 # of buffered whole in memory (see the docstring note above).
344 # Accept-Encoding: identity mirrors what the old
345 # urlretrieve-based path effectively sent -- a best-effort
346 # signal only; it does NOT rule out a decompression bomb on
347 # its own (a server that ignores this header and gzips the
348 # body anyway is still transparently decoded downstream). The
349 # running byte count enforced while streaming below, over the
350 # already-decoded chunks, is what actually bounds that (see
351 # the docstring note above).
352 with session.get(
353 pdf_url,
354 timeout=DEFAULT_TIMEOUT,
355 allow_redirects=True,
356 stream=True,
357 headers={"Accept-Encoding": "identity"},
358 ) as response:
359 response.raise_for_status()
360 self._stream_response_to_file(response, target_path)
362 return str(target_path)
364 def _stream_response_to_file(
365 self, response: Any, target_path: Path
366 ) -> None:
367 """Write ``response``'s body to ``target_path`` in bounded chunks.
369 Streams via ``iter_content`` rather than reading ``response.content``
370 (which would materialise the whole body in memory) -- see
371 ``_download_pdf_safely``'s docstring. ``written`` is an independent
372 running cap: it is what actually bounds this fetch's memory/disk
373 use, not the ``Content-Length`` check in ``SafeSession`` (see that
374 docstring for why the check alone is not enough here).
376 The body is streamed to a per-call-unique ``<target
377 name>.<uuid4 hex>.part`` sibling and only ``os.replace()``d onto
378 ``target_path`` after a full, under-cap, *non-empty* write -- never
379 streamed directly into ``target_path``. Two concurrent downloads of
380 the same arXiv id into the same directory each get their own
381 ``.part`` file, so one failing (e.g. tripping the size cap) can
382 only ever remove *its own* partial file, never a sibling
383 download's already-completed ``target_path``.
385 A response body that streams zero bytes (e.g. a ``200`` with an
386 empty body) is rejected before ``os.replace()`` runs, specifically
387 so it can never silently overwrite an already-completed
388 ``target_path`` from an earlier download of the same id with a
389 0-byte file while still reporting success.
391 Cleanup runs in a ``finally``, not an ``except Exception`` --
392 ``except Exception`` does not catch ``KeyboardInterrupt`` /
393 ``SystemExit`` (or a raw thread-kill), which would otherwise leave
394 an unpredictably-named, up-to-``MAX_RESPONSE_SIZE``-byte ``.part``
395 file orphaned in this shared root with no sweeper to reclaim it.
396 On success ``os.replace()`` has already moved ``tmp_path`` onto
397 ``target_path`` by the time ``finally`` runs, so the same
398 unconditional ``unlink(missing_ok=True)`` call is a no-op then;
399 on any failure -- including the cap being exceeded, an empty body,
400 or an interrupt -- it removes that call's own ``.part`` file. The
401 unlink itself is best-effort and never allowed to replace (mask)
402 whatever exception is already propagating.
403 """
404 written = 0
405 tmp_path = target_path.with_name(
406 f"{target_path.name}.{uuid.uuid4().hex}.part"
407 )
408 try:
409 # Public arXiv PDF (a published paper, no PII/secrets) fetched
410 # through the SSRF-validated SafeSession and written into the
411 # caller-provided, containment-checked download dir.
412 with tmp_path.open("wb") as pdf_file: # Safe: public PDF
413 for chunk in response.iter_content(chunk_size=1024 * 1024):
414 if not chunk: 414 ↛ 415line 414 didn't jump to line 415 because the condition on line 414 was never true
415 continue
416 written += len(chunk)
417 self._enforce_response_size_cap(written)
418 pdf_file.write(chunk)
419 if written == 0:
420 raise ValueError(
421 "arXiv PDF body was empty; refusing to write a "
422 "0-byte file over any existing download at the "
423 "target path"
424 )
425 os.replace(tmp_path, target_path)
426 finally:
427 try:
428 tmp_path.unlink(missing_ok=True)
429 except OSError:
430 # Never let a failure to clean up the partial file mask
431 # whatever exception (if any) is already propagating.
432 logger.debug(
433 f"Failed to remove partial download file {tmp_path.name}"
434 )
436 def _get_previews(self, query: str) -> List[Dict[str, Any]]:
437 """
438 Get preview information for arXiv papers.
440 Args:
441 query: The search query
443 Returns:
444 List of preview dictionaries
445 """
446 logger.info("Getting paper previews from arXiv")
448 try:
449 # Get search results from arXiv
450 papers = self._get_search_results(query)
452 # Store the paper objects for later use
453 self._papers = {paper.entry_id: paper for paper in papers}
455 # Format results as previews with basic information
456 previews = []
457 for paper in papers:
458 preview = {
459 "id": paper.entry_id, # Use entry_id as ID
460 "title": paper.title,
461 "link": paper.entry_id, # arXiv URL
462 "snippet": (
463 paper.summary[:SNIPPET_LENGTH_SHORT] + "..."
464 if len(paper.summary) > SNIPPET_LENGTH_SHORT
465 else paper.summary
466 ),
467 "authors": [
468 author.name for author in paper.authors[:3]
469 ], # First 3 authors
470 "published": (
471 paper.published.strftime("%Y-%m-%d")
472 if paper.published
473 else None
474 ),
475 "journal_ref": paper.journal_ref,
476 "source": "arXiv",
477 }
479 previews.append(preview)
481 return previews
483 except Exception as e:
484 error_msg = str(e)
485 safe_msg = self._scrub_error(e)
486 logger.exception(
487 f"Error getting arXiv previews ({type(e).__name__}): {safe_msg}"
488 )
490 # Check for rate limiting patterns
491 if (
492 "429" in error_msg
493 or "too many requests" in error_msg.lower()
494 or "rate limit" in error_msg.lower()
495 or "service unavailable" in error_msg.lower()
496 or "503" in error_msg
497 ):
498 # `from None` suppresses the implicit __context__ chain:
499 # the original exception still carries the raw message, so
500 # a full traceback render (chain=True) would re-leak the
501 # secret that safe_msg just scrubbed.
502 raise RateLimitError(
503 f"arXiv rate limit hit: {safe_msg}"
504 ) from None
506 return []
508 def _get_full_content(
509 self, relevant_items: List[Dict[str, Any]]
510 ) -> List[Dict[str, Any]]:
511 """
512 Get full content for the relevant arXiv papers.
513 Downloads PDFs and extracts text when include_full_text is True.
514 Limits the number of PDFs processed to max_full_text.
516 Args:
517 relevant_items: List of relevant preview dictionaries
519 Returns:
520 List of result dictionaries with full content
521 """
522 logger.info("Getting full content for relevant arXiv papers")
524 results = []
525 pdf_count = 0 # Track number of PDFs processed
527 for item in relevant_items:
528 # Start with the preview data
529 result = item.copy()
531 # Get the paper ID
532 paper_id = item.get("id")
534 # Try to get the full paper from our cache
535 paper = None
536 if hasattr(self, "_papers") and paper_id in self._papers:
537 paper = self._papers[paper_id]
539 if paper:
540 # Add complete paper information
541 result.update(
542 {
543 "pdf_url": paper.pdf_url,
544 "authors": [
545 author.name for author in paper.authors
546 ], # All authors
547 "published": (
548 paper.published.strftime("%Y-%m-%d")
549 if paper.published
550 else None
551 ),
552 "updated": (
553 paper.updated.strftime("%Y-%m-%d")
554 if paper.updated
555 else None
556 ),
557 "categories": paper.categories,
558 "summary": paper.summary, # Full summary
559 "comment": paper.comment,
560 "doi": paper.doi,
561 # Explicitly forward for journal quality filter
562 "journal_ref": paper.journal_ref,
563 }
564 )
566 # Default to using summary as content
567 result["content"] = paper.summary
568 result["full_content"] = paper.summary
570 # Download PDF and extract text if requested and within limit
571 if (
572 self.include_full_text
573 and self.download_dir
574 and pdf_count < self.max_full_text
575 ):
576 try:
577 # Download the paper
578 pdf_count += (
579 1 # Increment counter before attempting download
580 )
581 # Apply rate limiting before PDF download
582 self.rate_tracker.apply_rate_limit(self.engine_type)
584 paper_path = self._download_pdf_safely(
585 paper, self.download_dir
586 )
587 result["pdf_path"] = str(paper_path)
589 # Extract text from PDF
590 try:
591 # Try pypdf first
592 try:
593 from pypdf import PdfReader
595 with open(paper_path, "rb") as pdf_file:
596 pdf_reader = PdfReader(pdf_file)
597 pdf_text = ""
598 for page in pdf_reader.pages:
599 pdf_text += page.extract_text() + "\n\n"
601 if (
602 pdf_text.strip()
603 ): # Only use if we got meaningful text
604 result["content"] = pdf_text
605 result["full_content"] = pdf_text
606 logger.info(
607 "Successfully extracted text from PDF using pypdf"
608 )
609 except (ImportError, Exception) as e1:
610 # Fall back to pdfplumber
611 try:
612 import pdfplumber
614 with pdfplumber.open(paper_path) as pdf:
615 pdf_text = ""
616 for plumber_page in pdf.pages:
617 pdf_text += (
618 plumber_page.extract_text()
619 + "\n\n"
620 )
622 if ( 622 ↛ 713line 622 didn't jump to line 713
623 pdf_text.strip()
624 ): # Only use if we got meaningful text
625 result["content"] = pdf_text
626 result["full_content"] = pdf_text
627 logger.info(
628 "Successfully extracted text from PDF using pdfplumber"
629 )
630 except (ImportError, Exception) as e2:
631 safe_e1 = self._scrub_error(e1)
632 safe_e2 = self._scrub_error(e2)
633 logger.exception(
634 f"PDF text extraction failed ({type(e1).__name__}, then {type(e2).__name__}): {safe_e1}, then {safe_e2}"
635 )
636 logger.info(
637 "Using paper summary as content instead"
638 )
639 except Exception as e:
640 safe_msg = self._scrub_error(e)
641 logger.exception(
642 f"Error extracting text from PDF ({type(e).__name__}): {safe_msg}"
643 )
644 logger.info(
645 "Using paper summary as content instead"
646 )
647 except RequestException as e:
648 # ORDER MATTERS: this clause must precede the
649 # (DirectoryCreationSecurityError, OSError)
650 # clause below. requests' RequestException
651 # subclasses IOError, which *is* OSError, so a
652 # bare OSError catch would otherwise swallow
653 # every ConnectionError/Timeout/HTTPError and
654 # relabel a routine network failure as a
655 # containment error. Network errors carry no
656 # local path, so the normal scrubbed message is
657 # both safe and far more useful here.
658 safe_msg = self._scrub_error(e)
659 logger.exception(
660 f"Error downloading paper {paper.title} ({type(e).__name__}): {safe_msg}"
661 )
662 result["pdf_path"] = None
663 pdf_count -= 1 # Decrement counter if download fails
664 except (DirectoryCreationSecurityError, OSError) as e:
665 # Never interpolate either exception's message or
666 # any path -- only its type name, which is
667 # path-free and safe to include (unlike str(e)).
668 # DirectoryCreationSecurityError embeds the
669 # resolved data-directory path
670 # (directory_creation.py). OSError (e.g.
671 # FileExistsError/NotADirectoryError) is what
672 # create_directory's own `p.mkdir()` raises
673 # uncaught -- with the *same* resolved, absolute
674 # path in its message -- when download_dir
675 # collides with an existing file or has one as a
676 # parent component; a filesystem error surfacing
677 # from the streaming write below (e.g. a disk-full
678 # OSError) can carry a path too. logger.exception
679 # here reaches the research owner's browser via
680 # frontend_progress_sink -- _scrub_error does not
681 # scrub filesystem paths, so a static, path-free
682 # message is used for this whole exception family
683 # instead; the type name alone lets an operator
684 # tell a containment rejection apart from e.g. a
685 # disk-full OSError (a cert_verify OSError from a
686 # missing REQUESTS_CA_BUNDLE, say).
687 logger.exception(
688 f"Error downloading paper {paper.title} "
689 f"({type(e).__name__}): a filesystem or "
690 "directory-containment error occurred"
691 )
692 result["pdf_path"] = None
693 pdf_count -= 1 # Decrement counter if download fails
694 except Exception as e:
695 safe_msg = self._scrub_error(e)
696 logger.exception(
697 f"Error downloading paper {paper.title} ({type(e).__name__}): {safe_msg}"
698 )
699 result["pdf_path"] = None
700 pdf_count -= 1 # Decrement counter if download fails
701 elif (
702 self.include_full_text
703 and self.download_dir
704 and pdf_count >= self.max_full_text
705 ):
706 # Reached PDF limit
707 logger.info(
708 f"Maximum number of PDFs ({self.max_full_text}) reached. Skipping remaining PDFs."
709 )
710 result["content"] = paper.summary
711 result["full_content"] = paper.summary
713 results.append(result)
715 return results
717 def run(
718 self, query: str, research_context: Dict[str, Any] | None = None
719 ) -> List[Dict[str, Any]]:
720 """
721 Execute a search using arXiv with the two-phase approach.
723 Args:
724 query: The search query
725 research_context: Context from previous research to use.
727 Returns:
728 List of search results
729 """
730 logger.info("---Execute a search using arXiv---")
732 # Use the implementation from the parent class which handles all phases
733 results = super().run(query, research_context=research_context)
735 # Clean up
736 if hasattr(self, "_papers"):
737 del self._papers
739 return results
741 def get_paper_details(self, arxiv_id: str) -> Dict[str, Any]:
742 """
743 Get detailed information about a specific arXiv paper.
745 Args:
746 arxiv_id: arXiv ID of the paper (e.g., '2101.12345')
748 Returns:
749 Dictionary with paper information
750 """
751 try:
752 # Create the search client
753 client = arxiv.Client()
755 # Search for the specific paper
756 search = arxiv.Search(id_list=[arxiv_id], max_results=1)
758 # Apply rate limiting before fetching paper by ID
759 self._last_wait_time = self.rate_tracker.apply_rate_limit(
760 self.engine_type
761 )
763 # Get the paper
764 papers = list(client.results(search))
765 if not papers:
766 return {}
768 paper = papers[0]
770 # Format result based on config
771 result = {
772 "title": paper.title,
773 "link": paper.entry_id,
774 "snippet": (
775 paper.summary[:250] + "..."
776 if len(paper.summary) > 250
777 else paper.summary
778 ),
779 "authors": [
780 author.name for author in paper.authors[:3]
781 ], # First 3 authors
782 "journal_ref": paper.journal_ref,
783 }
785 result.update(
786 {
787 "pdf_url": paper.pdf_url,
788 "authors": [
789 author.name for author in paper.authors
790 ], # All authors
791 "published": (
792 paper.published.strftime("%Y-%m-%d")
793 if paper.published
794 else None
795 ),
796 "updated": (
797 paper.updated.strftime("%Y-%m-%d")
798 if paper.updated
799 else None
800 ),
801 "categories": paper.categories,
802 "summary": paper.summary, # Full summary
803 "comment": paper.comment,
804 "doi": paper.doi,
805 "content": paper.summary, # Use summary as content
806 "full_content": paper.summary, # For consistency
807 }
808 )
810 # Download PDF if requested
811 if self.include_full_text and self.download_dir:
812 try:
813 # Apply rate limiting before PDF download
814 self.rate_tracker.apply_rate_limit(self.engine_type)
816 # Download the paper through the SSRF-validated session
817 paper_path = self._download_pdf_safely(
818 paper, self.download_dir
819 )
820 result["pdf_path"] = str(paper_path)
821 except RequestException as e:
822 # ORDER MATTERS: this clause must precede the
823 # (DirectoryCreationSecurityError, OSError)
824 # clause below. requests' RequestException
825 # subclasses IOError, which *is* OSError, so a
826 # bare OSError catch would otherwise swallow
827 # every ConnectionError/Timeout/HTTPError and
828 # relabel a routine network failure as a
829 # containment error. Network errors carry no
830 # local path, so the normal scrubbed message is
831 # both safe and far more useful here.
832 safe_msg = self._scrub_error(e)
833 logger.exception(
834 f"Error downloading paper ({type(e).__name__}): {safe_msg}"
835 )
836 except (DirectoryCreationSecurityError, OSError) as e:
837 # Never interpolate either exception's message or any
838 # path -- only its type name, which is path-free and
839 # safe to include (unlike str(e)).
840 # DirectoryCreationSecurityError embeds the resolved
841 # data-directory path (directory_creation.py). OSError
842 # (e.g. FileExistsError/NotADirectoryError) is what
843 # create_directory's own `p.mkdir()` raises uncaught
844 # -- with the *same* resolved, absolute path in its
845 # message -- when download_dir collides with an
846 # existing file or has one as a parent component; a
847 # filesystem error surfacing from the streaming write
848 # below (e.g. a disk-full OSError) can carry a path
849 # too. logger.exception here reaches the research
850 # owner's browser via frontend_progress_sink --
851 # _scrub_error does not scrub filesystem paths, so a
852 # static, path-free message is used for this whole
853 # exception family instead; the type name alone lets
854 # an operator tell a containment rejection apart from
855 # e.g. a disk-full OSError (a cert_verify OSError from
856 # a missing REQUESTS_CA_BUNDLE, say).
857 logger.exception(
858 f"Error downloading paper ({type(e).__name__}): "
859 "a filesystem or directory-containment error "
860 "occurred"
861 )
862 except Exception as e:
863 safe_msg = self._scrub_error(e)
864 logger.exception(
865 f"Error downloading paper ({type(e).__name__}): {safe_msg}"
866 )
868 return result
870 except Exception as e:
871 safe_msg = self._scrub_error(e)
872 logger.exception(
873 f"Error getting paper details ({type(e).__name__}): {safe_msg}"
874 )
875 return {}
877 def search_by_author(
878 self, author_name: str, max_results: Optional[int] = None
879 ) -> List[Dict[str, Any]]:
880 """
881 Search for papers by a specific author.
883 Args:
884 author_name: Name of the author
885 max_results: Maximum number of results (defaults to self.max_results)
887 Returns:
888 List of papers by the author
889 """
890 original_max_results = self.max_results
892 try:
893 if max_results:
894 self.max_results = max_results
896 query = f'au:"{author_name}"'
897 return self.run(query)
899 finally:
900 # Restore original value
901 self.max_results = original_max_results
903 def search_by_category(
904 self, category: str, max_results: Optional[int] = None
905 ) -> List[Dict[str, Any]]:
906 """
907 Search for papers in a specific arXiv category.
909 Args:
910 category: arXiv category (e.g., 'cs.AI', 'physics.optics')
911 max_results: Maximum number of results (defaults to self.max_results)
913 Returns:
914 List of papers in the category
915 """
916 original_max_results = self.max_results
918 try:
919 if max_results:
920 self.max_results = max_results
922 query = f"cat:{category}"
923 return self.run(query)
925 finally:
926 # Restore original value
927 self.max_results = original_max_results