Coverage for src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py: 99%
207 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1import re
2from typing import Any, Dict, List, Optional
4from langchain_core.language_models import BaseLLM
5from requests.exceptions import RequestException
7from ...constants import SNIPPET_LENGTH_SHORT
8from ...security.secure_logging import logger
9from ...utilities.arxiv_api import (
10 ArxivIdRequest,
11 ArxivPaper,
12 ArxivQueryRequest,
13 ArxivSortCriterion,
14 ArxivSortOrder,
15 fetch_arxiv_results,
16)
17from ..rate_limiting import RateLimitError
18from ..engine_availability import retry_after_from_error
19from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
21# Canonical arXiv identifier shapes, anchored so nothing else slips through
22# before we build an egress URL from the value:
23# - new style: 2301.12345 / 2301.12345v2 (4-or-5 digit sequence)
24# - old style: math.GT/0309136 or cond-mat/0501234 (with optional version)
25_ARXIV_ID_RE = re.compile(
26 r"^(?:\d{4}\.\d{4,5}(?:v\d+)?|[a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?)$"
27)
30class FullTextNotExtractedError(Exception):
31 """The paper's PDF was downloaded but produced no text.
33 Raised by ``ArXivSearchEngine._fetch_full_text`` so the caller can tell
34 it apart from a failed fetch (``None``): the download and the
35 extraction were paid for, so the full-text budget stays spent.
36 """
39class FullTextFetchFailedError(Exception):
40 """A full-text request to arXiv failed instead of being answered.
42 Raised by ``ArXivSearchEngine._fetch_full_text`` when the paper's HTML
43 or PDF request timed out, could not connect, got a 5xx or other
44 unexpected status, or hit the decoded-body cap, and no text arrived.
45 Nothing says the paper lacks full text: the host is failing, so
46 ``_get_full_content`` makes no further full-text fetches for the rest
47 of that call instead of paying a request (and a timeout) per paper.
48 """
51class FullTextRateLimitedError(FullTextFetchFailedError):
52 """arXiv answered the paper's PDF request with HTTP 429 or 503.
54 Raised by ``ArXivSearchEngine._fetch_full_text``. The host is rate
55 limiting or unavailable: as for any ``FullTextFetchFailedError``,
56 ``_get_full_content`` makes no further full-text fetches for the rest
57 of that call instead of sending a request per paper.
58 """
61class ArXivSearchEngine(BaseSearchEngine):
62 """arXiv search engine implementation with two-phase approach"""
64 # Mark as public search engine
65 is_public = True
66 egress_sensitivity = Sensitivity.NON_SENSITIVE
67 egress_exposure = Exposure.EXPOSING
68 # Not a generic search engine (specialized for academic papers)
69 is_generic = False
70 # Scientific/academic search engine
71 is_scientific = True
72 is_lexical = True
73 needs_llm_relevance_filter = True
75 def __init__(
76 self,
77 max_results: int = 10,
78 sort_by: str = "relevance",
79 sort_order: str = "descending",
80 include_full_text: bool = False,
81 download_dir: Optional[str] = None,
82 max_full_text: int = 1,
83 llm: Optional[BaseLLM] = None,
84 max_filtered_results: Optional[int] = None,
85 settings_snapshot: Optional[Dict[str, Any]] = None,
86 ): # Added this parameter
87 """
88 Initialize the arXiv search engine.
90 Args:
91 max_results: Maximum number of search results
92 sort_by: Sorting criteria ('relevance', 'lastUpdatedDate', or 'submittedDate')
93 sort_order: Sort order ('ascending' or 'descending')
94 include_full_text: Whether to include full paper content in results (downloads PDF)
95 download_dir: Gates full-text fetching -- full text is only
96 attempted when include_full_text is True and this is set.
97 The value itself is not used: _fetch_full_text routes
98 through the shared HTML-first ArxivDownloader (issue #4783),
99 which keeps the text in memory and writes nothing to disk.
100 max_full_text: Maximum number of papers whose full text is
101 fetched (default: 1). A paper counts once full text or its
102 PDF arrived, even if the PDF yielded no text. A paper arXiv
103 answered has no full text (no rendition, and its PDF answered
104 404/410 or with no PDF) does not count, so a later paper
105 still gets the attempt. A request that failed instead of
106 being answered (timeout, connection error, 5xx, HTTP
107 429/503) counts, and stops full-text fetching for the
108 remaining papers of that search, so a failing host costs
109 one paper's requests rather than one per paper.
110 llm: Language model for relevance filtering
111 max_filtered_results: Maximum number of results to keep after filtering
112 settings_snapshot: Settings snapshot for thread context
113 """
114 # Initialize the journal reputation filter if needed.
115 # Runs as a preview filter (before LLM relevance) because Tiers 1-3
116 # are instant data lookups — no point sending irrelevant journals
117 # through the expensive LLM relevance filter.
118 preview_filters = []
119 journal_filter = self._create_journal_filter(
120 "arxiv", llm, settings_snapshot
121 )
122 if journal_filter is not None:
123 preview_filters.append(journal_filter)
125 super().__init__(
126 llm=llm,
127 max_filtered_results=max_filtered_results,
128 max_results=max_results,
129 preview_filters=preview_filters, # type: ignore[arg-type]
130 settings_snapshot=settings_snapshot,
131 )
132 self.max_results = max(self.max_results, 25)
133 self.sort_by = sort_by
134 self.sort_order = sort_order
135 self.include_full_text = include_full_text
136 self.download_dir = download_dir
137 self.max_full_text = max_full_text
139 self.sort_criteria = {
140 "relevance": ArxivSortCriterion.RELEVANCE,
141 "lastUpdatedDate": ArxivSortCriterion.LAST_UPDATED_DATE,
142 "submittedDate": ArxivSortCriterion.SUBMITTED_DATE,
143 }
145 self.sort_directions = {
146 "ascending": ArxivSortOrder.ASCENDING,
147 "descending": ArxivSortOrder.DESCENDING,
148 }
150 def _get_search_results(self, query: str) -> list[ArxivPaper]:
151 """
152 Helper method to get search results from arXiv API.
154 Args:
155 query: The search query
157 Returns:
158 List of arXiv paper objects
159 """
160 # Configure the search client
161 sort_criteria = self.sort_criteria.get(
162 self.sort_by, ArxivSortCriterion.RELEVANCE
163 )
164 sort_order = self.sort_directions.get(
165 self.sort_order, ArxivSortOrder.DESCENDING
166 )
168 request = ArxivQueryRequest(
169 query=query,
170 max_results=self.max_results,
171 sort_by=sort_criteria,
172 sort_order=sort_order,
173 )
175 # Apply rate limiting before making the request
176 self._last_wait_time = self.rate_tracker.apply_rate_limit(
177 self.engine_type
178 )
180 return fetch_arxiv_results(request)
182 @staticmethod
183 def _validated_arxiv_id(paper: Any) -> Optional[str]:
184 """Return a canonical, validated arXiv id for ``paper`` or ``None``.
186 The id is derived from the arxiv-provided ``entry_id`` (e.g.
187 ``http://arxiv.org/abs/2301.12345v1``) and matched against
188 ``_ARXIV_ID_RE`` before it is ever used to build an egress URL,
189 so a malformed or unexpected value cannot be interpolated into a
190 request target.
191 """
192 entry_id = getattr(paper, "entry_id", "") or ""
193 match = re.search(r"arxiv\.org/abs/(.+)$", entry_id)
194 candidate = (match.group(1) if match else entry_id).strip()
195 if _ARXIV_ID_RE.match(candidate):
196 return candidate
197 return None
199 def _fetch_full_text(self, paper: Any) -> Optional[str]:
200 """Fetch a paper's full text through the shared HTML-first downloader.
202 This routes ``ArXivSearchEngine``'s full-text step through
203 ``ArxivDownloader.download_full_text`` (issue #4783): the
204 equation-preserving official arXiv HTML rendition when one exists,
205 then the arXiv PDF, and never the API metadata leg that
206 ``download_text_with_source`` would add. That replaces the pypdf
207 extraction this engine used to inline.
209 The downloader is imported lazily: importing
210 ``research_library.downloaders.arxiv`` at module scope would
211 pull the whole download stack into every engine import, and
212 its base module imports back into
213 ``web_search_engines.rate_limiting``.
215 The downloader fetches from ``export.arxiv.org``, the host arXiv
216 sets aside for programmatic access, rather than ``arxiv.org``.
218 Returns the text only when it is full text: the HTML rendition or
219 text extracted from the PDF. Returns ``None`` when arXiv answered
220 that the paper has no full text (no usable rendition, and no PDF
221 bytes). The downloader handles its own transport errors on every
222 leg (HTTP errors, 429s, connection failures and the decoded-body
223 cap all come back as a status, not as an exception), which this
224 method turns into the exceptions below. The API metadata leg is
225 never requested (``download_full_text``): it yields the abstract,
226 which the engine already holds as the paper summary, so it would
227 add an arXiv API request without adding text.
229 Raises ``FullTextNotExtractedError`` when the PDF arrived but
230 produced no text (an image-only scan, or the extraction ceilings
231 hit first), ``FullTextRateLimitedError`` when arXiv answered the
232 PDF request with HTTP 429 or 503 (the PDF leg makes one attempt
233 and does not retry), ``FullTextFetchFailedError`` when a request
234 failed instead of being answered (timeout, connection error, 5xx,
235 the decoded-body cap), and ``ValueError`` when the paper carries
236 no valid arXiv id.
237 """
238 arxiv_id = self._validated_arxiv_id(paper)
239 if not arxiv_id:
240 raise ValueError("Could not derive a valid arXiv id for full text")
242 from ...research_library.downloaders.arxiv import (
243 ARXIV_EXPORT_HOST,
244 ArxivDownloader,
245 ArxivFullTextStatus,
246 ArxivTextSource,
247 )
249 downloader = getattr(self, "_arxiv_text_downloader", None)
250 if downloader is None:
251 downloader = ArxivDownloader(fetch_host=ARXIV_EXPORT_HOST)
252 self._arxiv_text_downloader = downloader
253 outcome = downloader.download_full_text(
254 f"https://arxiv.org/abs/{arxiv_id}"
255 )
256 if outcome.status == ArxivFullTextStatus.PDF_WITHOUT_TEXT:
257 raise FullTextNotExtractedError(
258 f"arXiv PDF for {arxiv_id} yielded no text"
259 )
260 if outcome.status == ArxivFullTextStatus.RATE_LIMITED:
261 raise FullTextRateLimitedError(
262 f"arXiv rate limited the PDF request for {arxiv_id}"
263 )
264 if outcome.status == ArxivFullTextStatus.FETCH_FAILED:
265 raise FullTextFetchFailedError(
266 f"arXiv full-text request failed for {arxiv_id}"
267 )
268 result = outcome.result
269 if result is None or result.source not in (
270 ArxivTextSource.ARXIV_HTML,
271 ArxivTextSource.LOCAL_PDF,
272 ):
273 return None
274 return result.text
276 def close(self) -> None:
277 """Close the lazily-created full-text downloader, then the base
278 engine's own resources.
280 ``_fetch_full_text`` caches an ``ArxivDownloader`` (which owns an
281 HTTP session) on ``self._arxiv_text_downloader`` the first time
282 full text is fetched; it is only ever created there, so nothing
283 else in this engine would release it. The attribute is cleared after
284 closing, so reuse after close() starts from a fresh downloader.
285 """
286 from ...utilities.resource_utils import safe_close
288 downloader = getattr(self, "_arxiv_text_downloader", None)
289 if downloader is not None:
290 safe_close(downloader, "arXiv text downloader")
291 # Drop the closed instance so a later full-text fetch builds a
292 # fresh downloader instead of reusing one without a session.
293 del self._arxiv_text_downloader
294 super().close()
296 def _log_full_text_error(
297 self, error: Exception, title: Optional[str] = None
298 ) -> None:
299 """Log an error raised by the full-text step without leaking paths.
301 An ``OSError`` message can carry a local filesystem path, which
302 ``_scrub_error`` does not remove, and ``logger.exception`` output
303 reaches the research owner's browser via ``frontend_progress_sink``.
304 So an ``OSError`` is logged by its type name only (which is
305 path-free and still tells an operator what kind of failure it
306 was). ``requests``' ``RequestException`` subclasses ``OSError`` but
307 carries no local path, so it keeps the scrubbed message, as does
308 every other exception.
309 """
310 paper = f"paper {title} " if title is not None else "paper "
311 if isinstance(error, OSError) and not isinstance(
312 error, RequestException
313 ):
314 logger.exception(
315 f"Error downloading {paper}({type(error).__name__}): "
316 "a filesystem error occurred"
317 )
318 return
319 safe_msg = self._scrub_error(error)
320 logger.exception(
321 f"Error downloading {paper}({type(error).__name__}): {safe_msg}"
322 )
324 def _get_previews(self, query: str) -> List[Dict[str, Any]]:
325 """
326 Get preview information for arXiv papers.
328 Args:
329 query: The search query
331 Returns:
332 List of preview dictionaries
333 """
334 logger.info("Getting paper previews from arXiv")
336 try:
337 # Get search results from arXiv
338 papers = self._get_search_results(query)
340 # Store the paper objects for later use
341 self._papers = {paper.entry_id: paper for paper in papers}
343 # Format results as previews with basic information
344 previews = []
345 for paper in papers:
346 preview = {
347 "id": paper.entry_id, # Use entry_id as ID
348 "title": paper.title,
349 "link": paper.entry_id, # arXiv URL
350 "snippet": (
351 paper.summary[:SNIPPET_LENGTH_SHORT] + "..."
352 if len(paper.summary) > SNIPPET_LENGTH_SHORT
353 else paper.summary
354 ),
355 "authors": [
356 author.name for author in paper.authors[:3]
357 ], # First 3 authors
358 "published": (
359 paper.published.strftime("%Y-%m-%d")
360 if paper.published
361 else None
362 ),
363 "journal_ref": paper.journal_ref,
364 "source": "arXiv",
365 }
367 previews.append(preview)
369 return previews
371 except Exception as e:
372 self._record_search_failure(e)
373 error_msg = str(e)
374 safe_msg = self._scrub_error(e)
375 logger.exception(
376 f"Error getting arXiv previews ({type(e).__name__}): {safe_msg}"
377 )
379 # Check for rate limiting patterns
380 if (
381 "429" in error_msg
382 or "too many requests" in error_msg.lower()
383 or "rate limit" in error_msg.lower()
384 or "service unavailable" in error_msg.lower()
385 or "503" in error_msg
386 ):
387 # `from None` suppresses the implicit __context__ chain:
388 # the original exception still carries the raw message, so
389 # a full traceback render (chain=True) would re-leak the
390 # secret that safe_msg just scrubbed.
391 raise RateLimitError(
392 f"arXiv rate limit hit: {safe_msg}",
393 retry_after=retry_after_from_error(e),
394 ) from None
396 return []
398 def _get_full_content(
399 self, relevant_items: List[Dict[str, Any]]
400 ) -> List[Dict[str, Any]]:
401 """
402 Get full content for the relevant arXiv papers.
403 Fetches full text through the shared HTML-first arXiv downloader
404 when include_full_text is True. At most max_full_text papers
405 have their full text fetched; a paper arXiv answered has no full
406 text is refunded, and a failed request stops further fetches (see
407 the constructor's max_full_text).
409 Args:
410 relevant_items: List of relevant preview dictionaries
412 Returns:
413 List of result dictionaries with full content
414 """
415 logger.info("Getting full content for relevant arXiv papers")
417 results = []
418 # Papers whose full-text fetch counts against max_full_text: full
419 # text or the PDF arrived (text or not), or a request failed. Only
420 # a paper arXiv answered has no full text is refunded below.
421 pdf_count = 0
422 # Set once a full-text request fails instead of being answered
423 # (timeout, connection error, 5xx, 429/503): no further full-text
424 # fetches are made in this call, so a failing host is not sent a
425 # request (and a timeout) per remaining paper.
426 host_failing = False
428 for item in relevant_items:
429 # Start with the preview data
430 result = item.copy()
432 # Get the paper ID
433 paper_id = item.get("id")
435 # Try to get the full paper from our cache
436 paper = None
437 if hasattr(self, "_papers") and paper_id in self._papers:
438 paper = self._papers[paper_id]
440 if paper:
441 arxiv_id = self._validated_arxiv_id(paper)
442 # Add complete paper information
443 result.update(
444 {
445 "pdf_url": (
446 f"https://arxiv.org/pdf/{arxiv_id}"
447 if arxiv_id
448 else None
449 ),
450 "authors": [
451 author.name for author in paper.authors
452 ], # All authors
453 "published": (
454 paper.published.strftime("%Y-%m-%d")
455 if paper.published
456 else None
457 ),
458 "updated": (
459 paper.updated.strftime("%Y-%m-%d")
460 if paper.updated
461 else None
462 ),
463 "categories": paper.categories,
464 "summary": paper.summary, # Full summary
465 "comment": paper.comment,
466 "doi": paper.doi,
467 # Explicitly forward for journal quality filter
468 "journal_ref": paper.journal_ref,
469 }
470 )
472 # Default to using summary as content
473 result["content"] = paper.summary
474 result["full_content"] = paper.summary
475 result["full_content_is_complete"] = False
477 # Fetch full text through the shared HTML-first arXiv
478 # downloader (official HTML rendition with a PDF
479 # fallback, issue #4783) when requested and within limit
480 if (
481 self.include_full_text
482 and self.download_dir
483 and pdf_count < self.max_full_text
484 and not host_failing
485 ):
486 pdf_count += 1 # Count the fetch before attempting it
487 full_text = None
488 fetched = False
489 try:
490 # Apply rate limiting before the full-text fetch
491 self.rate_tracker.apply_rate_limit(self.engine_type)
493 full_text = self._fetch_full_text(paper)
494 fetched = full_text is not None
495 except FullTextNotExtractedError:
496 # The PDF arrived but yielded no text: the download
497 # and extraction were paid for, so the fetch counts.
498 fetched = True
499 logger.info(
500 "arXiv PDF yielded no text; keeping the summary"
501 )
502 except FullTextFetchFailedError as e:
503 # The host is failing (429/503, timeout, connection
504 # error, 5xx): the fetch counts against the budget
505 # and no further full text is fetched in this call.
506 fetched = True
507 host_failing = True
508 if isinstance(e, FullTextRateLimitedError):
509 logger.warning(
510 "arXiv rate limited the full-text fetch; "
511 "using summaries for the remaining papers"
512 )
513 else:
514 logger.warning(
515 "arXiv full-text fetch failed; using "
516 "summaries for the remaining papers"
517 )
518 except Exception as e:
519 # The downloader handles its own transport errors
520 # (see _fetch_full_text; a failed request comes
521 # back as FullTextFetchFailedError, handled
522 # above), so only errors outside
523 # the transport reach this point: an invalid arXiv
524 # id, the rate limiter, or a bug. None of them
525 # fetched anything.
526 self._log_full_text_error(e, paper.title)
527 if full_text and full_text.strip():
528 result["content"] = full_text
529 result["full_content"] = full_text
530 result["full_content_is_complete"] = True
531 logger.info(
532 "Retrieved full text via the HTML-first arXiv downloader"
533 )
534 else:
535 if not fetched:
536 # arXiv answered that the paper has no full
537 # text, or an error stopped the fetch before
538 # any request: refund the budget so a later
539 # paper still gets a full-text attempt.
540 pdf_count -= 1
541 logger.info("Using paper summary as content instead")
542 elif (
543 self.include_full_text
544 and self.download_dir
545 and pdf_count >= self.max_full_text
546 ):
547 # Reached full-text fetch limit
548 logger.info(
549 f"Maximum number of full-text fetches ({self.max_full_text}) reached. Skipping remaining papers."
550 )
551 result["content"] = paper.summary
552 result["full_content"] = paper.summary
553 result["full_content_is_complete"] = False
555 results.append(result)
557 return results
559 def run(
560 self, query: str, research_context: Dict[str, Any] | None = None
561 ) -> List[Dict[str, Any]]:
562 """
563 Execute a search using arXiv with the two-phase approach.
565 Args:
566 query: The search query
567 research_context: Context from previous research to use.
569 Returns:
570 List of search results
571 """
572 logger.info("---Execute a search using arXiv---")
574 # Use the implementation from the parent class which handles all phases
575 results = super().run(query, research_context=research_context)
577 # Clean up
578 if hasattr(self, "_papers"):
579 del self._papers
581 return results
583 def get_paper_details(self, arxiv_id: str) -> Dict[str, Any]:
584 """
585 Get detailed information about a specific arXiv paper.
587 Args:
588 arxiv_id: arXiv ID of the paper (e.g., '2101.12345')
590 Returns:
591 Dictionary with paper information
592 """
593 try:
594 # Apply rate limiting before fetching paper by ID
595 self._last_wait_time = self.rate_tracker.apply_rate_limit(
596 self.engine_type
597 )
599 papers = fetch_arxiv_results(ArxivIdRequest(arxiv_id=arxiv_id))
600 if not papers:
601 return {}
603 paper = papers[0]
605 # Format result based on config
606 result: Dict[str, Any] = {
607 "title": paper.title,
608 "link": paper.entry_id,
609 "snippet": (
610 paper.summary[:250] + "..."
611 if len(paper.summary) > 250
612 else paper.summary
613 ),
614 "authors": [
615 author.name for author in paper.authors[:3]
616 ], # First 3 authors
617 "journal_ref": paper.journal_ref,
618 }
620 canonical_id = self._validated_arxiv_id(paper)
621 result.update(
622 {
623 "pdf_url": (
624 f"https://arxiv.org/pdf/{canonical_id}"
625 if canonical_id
626 else None
627 ),
628 "authors": [
629 author.name for author in paper.authors
630 ], # All authors
631 "published": (
632 paper.published.strftime("%Y-%m-%d")
633 if paper.published
634 else None
635 ),
636 "updated": (
637 paper.updated.strftime("%Y-%m-%d")
638 if paper.updated
639 else None
640 ),
641 "categories": paper.categories,
642 "summary": paper.summary, # Full summary
643 "comment": paper.comment,
644 "doi": paper.doi,
645 "content": paper.summary, # Use summary as content
646 "full_content": paper.summary, # For consistency
647 "full_content_is_complete": False,
648 }
649 )
651 # Fetch full text through the shared HTML-first downloader
652 # (official HTML rendition with a PDF fallback, #4783)
653 # if requested
654 if self.include_full_text and self.download_dir:
655 try:
656 # Apply rate limiting before the full-text fetch
657 self.rate_tracker.apply_rate_limit(self.engine_type)
659 full_text = self._fetch_full_text(paper)
660 if full_text and full_text.strip(): 660 ↛ 694line 660 didn't jump to line 694 because the condition on line 660 was always true
661 result["content"] = full_text
662 result["full_content"] = full_text
663 result["full_content_is_complete"] = True
664 logger.info(
665 "Retrieved full text via the HTML-first arXiv downloader"
666 )
667 except FullTextNotExtractedError:
668 logger.info(
669 "arXiv PDF yielded no text; keeping the summary"
670 )
671 except FullTextRateLimitedError:
672 # arXiv answered the PDF request with 429/503: nothing
673 # arrived, so keep the summary. Not an error in this
674 # code, so no traceback.
675 logger.warning(
676 "arXiv rate limited the full-text fetch; keeping "
677 "the summary"
678 )
679 except FullTextFetchFailedError:
680 # A request timed out, could not connect or got a 5xx:
681 # nothing arrived, so keep the summary. Logged by the
682 # downloader as a warning; not an error here either.
683 logger.warning(
684 "arXiv full-text fetch failed; keeping the summary"
685 )
686 except Exception as e:
687 # Failed requests (429/503, timeouts, connection
688 # errors, 5xx) are handled above (see
689 # _fetch_full_text), so only errors outside
690 # the transport reach this point: an invalid arXiv id,
691 # the rate limiter, or a bug.
692 self._log_full_text_error(e)
694 return result
696 except Exception as e:
697 safe_msg = self._scrub_error(e)
698 logger.exception(
699 f"Error getting paper details ({type(e).__name__}): {safe_msg}"
700 )
701 return {}
703 def search_by_author(
704 self, author_name: str, max_results: Optional[int] = None
705 ) -> List[Dict[str, Any]]:
706 """
707 Search for papers by a specific author.
709 Args:
710 author_name: Name of the author
711 max_results: Maximum number of results (defaults to self.max_results)
713 Returns:
714 List of papers by the author
715 """
716 original_max_results = self.max_results
718 try:
719 if max_results:
720 self.max_results = max_results
722 query = f'au:"{author_name}"'
723 return self.run(query)
725 finally:
726 # Restore original value
727 self.max_results = original_max_results
729 def search_by_category(
730 self, category: str, max_results: Optional[int] = None
731 ) -> List[Dict[str, Any]]:
732 """
733 Search for papers in a specific arXiv category.
735 Args:
736 category: arXiv category (e.g., 'cs.AI', 'physics.optics')
737 max_results: Maximum number of results (defaults to self.max_results)
739 Returns:
740 List of papers in the category
741 """
742 original_max_results = self.max_results
744 try:
745 if max_results:
746 self.max_results = max_results
748 query = f"cat:{category}"
749 return self.run(query)
751 finally:
752 # Restore original value
753 self.max_results = original_max_results