Coverage for src/local_deep_research/utilities/arxiv.py: 100%

1"""Shared arXiv URL identifier parsing.""" 

2 

3import re 

4from typing import Final, NewType 

5from urllib.parse import urlparse 

6 

7from .host_match import http_authority_host, is_host_or_subdomain 

8 

9ArxivId = NewType("ArxivId", str) 

10 

11_ID_PATTERN: Final = ( 

12 r"(?:\d{4}\.\d{4,5}|[A-Za-z]+(?:-[A-Za-z]+)*(?:\.[A-Za-z]+)?/\d{7})" 

13 r"(?:v\d+)?" 

14) 

15_ARXIV_PATH: Final = re.compile( 

16 rf"^/(?:abs|pdf|html)/(?P<identifier>{_ID_PATTERN})(?:\.pdf)?/?$", 

17 re.IGNORECASE, 

18) 

19_AR5IV_DIRECT_PATH: Final = re.compile( 

20 rf"^/(?P<identifier>{_ID_PATTERN})(?:\.pdf)?/?$", 

21 re.IGNORECASE, 

22) 

23_ARXIV_FAMILY_DOMAINS: Final = ("arxiv.org", "ar5iv.org") 

24 

25 

26def _normalize_http_hostname(scheme: str, hostname: str | None) -> str | None: 

27 if scheme.lower() not in {"http", "https"} or hostname is None: 

28 return None 

29 return hostname.lower().rstrip(".") 

30 

31 

32def _normalized_http_hostname(url: str) -> str | None: 

33 try: 

34 parsed = urlparse(url) 

35 except ValueError: 

36 return None 

37 if parsed.scheme.lower() not in {"http", "https"}: 

38 return None 

39 return http_authority_host(parsed) 

40 

41 

42def is_arxiv_family_url(url: str) -> bool: 

43 """Return whether an HTTP(S) URL is on an arXiv-family host. 

44 

45 This is host membership only, and the family includes the bare 

46 ``ar5iv.org`` domain, which arXiv does not serve. That makes it the right 

47 gate for deciding whether a *supplied* URL is worth parsing for an 

48 identifier (:func:`extract_arxiv_id` uses it for exactly that) and the 

49 wrong one for deciding whether a *response* came from arXiv: see 

50 ``ArxivDownloader._is_matching_arxiv_paper_url``, which deliberately 

51 tests the narrower ``.arxiv.org`` boundary so ``ar5iv.org`` can be an 

52 input identifier without ever being treated as authoritative. 

53 """ 

54 hostname = _normalized_http_hostname(url) 

55 return hostname is not None and any( 

56 is_host_or_subdomain(hostname, domain) 

57 for domain in _ARXIV_FAMILY_DOMAINS 

58 ) 

59 

60 

61def is_ar5iv_url(url: str) -> bool: 

62 """Return whether an HTTP(S) URL targets the ar5iv host family.""" 

63 hostname = _normalized_http_hostname(url) 

64 return hostname is not None and is_host_or_subdomain(hostname, "ar5iv.org") 

65 

66 

67def is_arxiv_paper_url(url: str) -> bool: 

68 """Return whether an HTTP(S) URL identifies one specific arXiv paper. 

69 

70 Host membership alone (:func:`is_arxiv_family_url`) is not enough to hand 

71 a URL to the arXiv downloader: ``arxiv.org/list/cs.AI/recent``, 

72 ``info.arxiv.org/help/...`` and ``arxiv.org/ftp/.../2301.12345.pdf`` are 

73 all on the family hosts but carry no identifier the downloader can act 

74 on, so they belong to the generic pipelines. Use this predicate for every 

75 routing and ownership decision. Response-URL trust is a separate and 

76 narrower question, answered by 

77 ``ArxivDownloader._is_matching_arxiv_paper_url`` rather than by either 

78 predicate here. 

79 """ 

80 return extract_arxiv_id(url) is not None 

81 

82 

83def extract_arxiv_id(url: str) -> ArxivId | None: 

84 """Return a version-preserving ID from a supported arXiv-family URL.""" 

85 try: 

86 parsed = urlparse(url) 

87 _ = parsed.port 

88 hostname = _normalize_http_hostname(parsed.scheme, parsed.hostname) 

89 except ValueError: 

90 return None 

91 

92 if hostname is None: 

93 return None 

94 

95 if not is_arxiv_family_url(url): 

96 return None 

97 is_ar5iv = is_ar5iv_url(url) 

98 

99 match = _ARXIV_PATH.fullmatch(parsed.path) 

100 if match is None and is_ar5iv: 

101 match = _AR5IV_DIRECT_PATH.fullmatch(parsed.path) 

102 if match is None: 

103 return None 

104 return ArxivId(match.group("identifier"))