Coverage for src/local_deep_research/content_fetcher/url_classifier.py: 97%
95 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2URL Classifier for content fetching.
4Determines the type of URL to route to the appropriate downloader.
5"""
7import re
8from enum import Enum
9from typing import Optional
10from urllib.parse import ParseResult, urlparse
12from ..utilities.arxiv import (
13 extract_arxiv_id as extract_arxiv_id_from_url,
14 is_arxiv_paper_url,
15)
16from ..utilities.host_match import http_authority_host, is_host_or_subdomain
19class URLType(Enum):
20 """Types of URLs we can handle."""
22 ARXIV = "arxiv"
23 PUBMED = "pubmed"
24 PMC = "pmc" # PubMed Central
25 SEMANTIC_SCHOLAR = "semantic_scholar"
26 BIORXIV = "biorxiv"
27 MEDRXIV = "medrxiv"
28 DOI = "doi"
29 PDF = "pdf" # Direct PDF link
30 HTML = "html" # Generic web page
31 INVALID = "invalid" # Dangerous/unsupported URL schemes
34# URL schemes that should be rejected (security risk)
35DANGEROUS_SCHEMES = {"javascript", "data", "file", "vbscript", "about"}
38class URLClassifier:
39 """Classifies URLs to determine the appropriate downloader."""
41 # Each source is a list of (host, path) pairs: the URL's host must be that
42 # host or a subdomain of it, and its path must start with the pattern.
43 # Both are read from the parsed URL. The patterns used to be searched for
44 # anywhere in the URL string, which also found them in another site's
45 # path or query (https://evil.example/doi.org/10.1234/x) and inside
46 # look-alike hosts (fakedoi.org), so such a URL was routed to the source's
47 # downloader and cited under its trusted tag (#6835).
48 # Note: paths are matched lowercased, as the whole URL was before.
50 # PubMed patterns
51 PUBMED_PATTERNS = [
52 ("pubmed.ncbi.nlm.nih.gov", r"/\d+"),
53 ("ncbi.nlm.nih.gov", r"/pubmed/\d+"),
54 ]
56 # PMC patterns (PubMed Central - full text)
57 PMC_PATTERNS = [
58 ("ncbi.nlm.nih.gov", r"/pmc/articles/pmc"),
59 ("europepmc.org", r"/article/pmc"),
60 ("europepmc.org", r"/articles/pmc"),
61 ]
63 # Semantic Scholar patterns
64 SEMANTIC_SCHOLAR_PATTERNS = [
65 ("semanticscholar.org", r"/paper/"),
66 ("api.semanticscholar.org", r"/"),
67 ]
69 # BioRxiv/MedRxiv patterns
70 BIORXIV_PATTERNS = [
71 ("biorxiv.org", r"/content/"),
72 ]
74 MEDRXIV_PATTERNS = [
75 ("medrxiv.org", r"/content/"),
76 ]
78 # DOI patterns (dx.doi.org is a subdomain of doi.org)
79 DOI_PATTERNS = [
80 ("doi.org", r"/10\."),
81 ]
83 @classmethod
84 def classify(cls, url: str) -> URLType:
85 """
86 Classify a URL to determine its type.
88 Args:
89 url: The URL to classify
91 Returns:
92 URLType enum indicating the type of content
93 """
94 url_lower = url.lower().strip()
96 # Check for dangerous URL schemes first (security)
97 try:
98 parsed = urlparse(url_lower)
99 if parsed.scheme in DANGEROUS_SCHEMES:
100 return URLType.INVALID
101 # Only allow http/https schemes
102 if parsed.scheme and parsed.scheme not in ("http", "https", ""):
103 return URLType.INVALID
104 except Exception:
105 return URLType.INVALID
107 # Only a URL that names one paper belongs to arXiv. Other pages on
108 # the family hosts (listings, author pages, info./blog./static.
109 # subdomains, the legacy /ftp/ PDF layout) carry no identifier the
110 # arXiv downloader can act on, and ARXIV ownership is terminal for
111 # every fetch entry point, so they stay with the generic pipelines.
112 if is_arxiv_paper_url(url.strip()):
113 return URLType.ARXIV
115 # Check direct PDF links after arXiv-family routing
116 if cls._is_pdf_url(url_lower):
117 return URLType.PDF
119 # Check specific academic sources
120 host, path = cls._host_and_path(parsed, url_lower)
121 for url_type, patterns in (
122 (URLType.PMC, cls.PMC_PATTERNS),
123 (URLType.PUBMED, cls.PUBMED_PATTERNS),
124 (URLType.SEMANTIC_SCHOLAR, cls.SEMANTIC_SCHOLAR_PATTERNS),
125 (URLType.BIORXIV, cls.BIORXIV_PATTERNS),
126 (URLType.MEDRXIV, cls.MEDRXIV_PATTERNS),
127 (URLType.DOI, cls.DOI_PATTERNS),
128 ):
129 for domain, path_pattern in patterns:
130 if is_host_or_subdomain(host, domain) and re.match(
131 path_pattern, path
132 ):
133 return url_type
135 # Default to HTML for web pages
136 return URLType.HTML
138 @staticmethod
139 def _host_and_path(parsed: ParseResult, url_lower: str) -> tuple[str, str]:
140 """The host and path the source patterns are matched against.
142 A URL without a scheme ("doi.org/10.1234/x"), or with too few slashes
143 after it ("https:/doi.org/10.1234/x"), passes the scheme check in
144 ``classify``, and ``urlparse`` reads its host as part of the path. So
145 what follows the scheme is read again as a network-path reference,
146 which is how browsers read it. An authority the HTTP stack would read
147 differently has no host (``http_authority_host``) and matches nothing.
148 """
149 if not parsed.netloc:
150 rest = url_lower.split(":", 1)[1] if parsed.scheme else url_lower
151 try:
152 parsed = urlparse("//" + rest.lstrip("/"))
153 except ValueError:
154 return "", ""
155 return http_authority_host(parsed) or "", parsed.path
157 @classmethod
158 def _is_pdf_url(cls, url: str) -> bool:
159 """Check if URL points directly to a PDF.
161 Note: Academic source PDFs (arXiv, bioRxiv, etc.) are handled
162 by their respective patterns, not as generic PDFs.
163 """
164 url_lower = url.lower()
166 # Academic sources with PDF URLs should use their specialized downloaders
167 academic_domains = [
168 "arxiv.org",
169 "biorxiv.org",
170 "medrxiv.org",
171 "ncbi.nlm.nih.gov",
172 "europepmc.org",
173 "semanticscholar.org",
174 ]
175 if any(domain in url_lower for domain in academic_domains):
176 return False
178 parsed = urlparse(url_lower)
179 path = parsed.path
181 # Check file extension
182 if path.endswith(".pdf"):
183 return True
185 # Check common PDF URL patterns
186 if "/pdf/" in path:
187 return True
189 return False
191 @classmethod
192 def extract_id(
193 cls, url: str, url_type: Optional[URLType] = None
194 ) -> Optional[str]:
195 """
196 Extract the identifier from a URL.
198 Args:
199 url: The URL to extract from
200 url_type: Optional pre-classified type
202 Returns:
203 Extracted ID or None
204 """
205 if url_type is None:
206 url_type = cls.classify(url)
208 if url_type == URLType.ARXIV:
209 return extract_arxiv_id_from_url(url)
211 if url_type == URLType.PUBMED:
212 # Extract PMID
213 match = re.search(r"/(\d+)/?", url)
214 if match: 214 ↛ 237line 214 didn't jump to line 237 because the condition on line 214 was always true
215 return match.group(1)
217 elif url_type == URLType.PMC:
218 # Extract PMC ID
219 match = re.search(r"PMC(\d+)", url, re.IGNORECASE)
220 if match:
221 return f"PMC{match.group(1)}"
223 elif url_type == URLType.SEMANTIC_SCHOLAR:
224 # Extract paper ID (40-char hex)
225 match = re.search(
226 r"/paper/(?:[^/]+/)?([a-f0-9]{40})", url, re.IGNORECASE
227 )
228 if match:
229 return match.group(1)
231 elif url_type == URLType.DOI:
232 # Extract DOI
233 match = re.search(r"(10\.\d{4,}/[^\s]+)", url)
234 if match: 234 ↛ 237line 234 didn't jump to line 237 because the condition on line 234 was always true
235 return match.group(1)
237 return None
239 @classmethod
240 def get_source_name(cls, url_type: URLType) -> str:
241 """Get human-readable source name."""
242 names = {
243 URLType.ARXIV: "arXiv",
244 URLType.PUBMED: "PubMed",
245 URLType.PMC: "PubMed Central",
246 URLType.SEMANTIC_SCHOLAR: "Semantic Scholar",
247 URLType.BIORXIV: "bioRxiv",
248 URLType.MEDRXIV: "medRxiv",
249 URLType.DOI: "DOI",
250 URLType.PDF: "PDF",
251 URLType.HTML: "Web Page",
252 URLType.INVALID: "Invalid URL",
253 }
254 return names.get(url_type, "Unknown")