Coverage for src/local_deep_research/research_library/downloaders/generic.py: 98%
90 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Generic PDF Downloader for unspecified sources
3"""
5from typing import Dict, Optional
6import requests
7from urllib.parse import urlparse, urlunparse
8from loguru import logger
10from ...security.ssrf_validator import redact_url_for_log
11from .base import BaseDownloader, ContentType, DownloadResult
14HTML_NOT_PDF_REASON = "Source is an HTML page, not a PDF"
17def _with_pdf_suffix(url: str) -> Optional[str]:
18 """Return ``url`` with ``.pdf`` appended to its path, or ``None`` if
19 no suffix is needed / the URL cannot be parsed.
21 Three edge cases the previous implementation got wrong:
23 1. **Bare host.** A URL like ``https://pmc.ncbi.nlm.nih.gov/`` has an
24 empty path. Naive string concatenation
25 (``url.rstrip("/") + ".pdf"``) produced
26 ``https://pmc.ncbi.nlm.nih.gov.pdf`` -- ``.pdf`` ended up appended
27 to the hostname, the SSRF validator rejected it as an unresolvable
28 hostname, and the URL was logged at ``ERROR`` even though the
29 fallback was a normal part of the download path.
30 2. **Trailing slash on a path.** ``https://example.com/paper/`` ended
31 up as ``https://example.com/paper.pdf`` by accident; that worked,
32 but only because ``rstrip("/")`` happened to delete exactly the
33 slash before appending.
34 3. **Query string.** ``https://example.com/paper?q=1`` became
35 ``https://example.com/paper?q=1.pdf``, putting ``.pdf`` after the
36 query separator and corrupting the URL.
38 We rebuild the URL via ``urlunparse(parsed._replace(path=...))`` so
39 the suffix lands in the path component and scheme/netloc/query/
40 fragment/userinfo are preserved verbatim. An empty path becomes
41 ``/index.pdf`` -- a deterministic placeholder that still describes
42 a resource at the host root.
44 The ``.pdf`` guard is case-insensitive and runs on the trailing
45 slashes-stripped path, so ``paper.pdf/``, ``paper.PDF``, and
46 ``paper.Pdf/`` are all recognised as already-suffixed and short-
47 circuit to ``None`` instead of producing ``paper.pdf.pdf``.
48 """
49 try:
50 parsed = urlparse(url)
51 except ValueError:
52 return None
54 stripped_path = parsed.path.rstrip("/") if parsed.path else ""
55 if stripped_path.lower().endswith(".pdf"):
56 return None
58 new_path = (stripped_path or "/index") + ".pdf"
59 return urlunparse(parsed._replace(path=new_path))
62class GenericDownloader(BaseDownloader):
63 """Generic downloader for any URL - attempts basic PDF download."""
65 def can_handle(self, url: str) -> bool:
66 """Generic downloader can handle any URL as a fallback."""
67 return True
69 def download(
70 self, url: str, content_type: ContentType = ContentType.PDF
71 ) -> Optional[bytes]:
72 """Attempt to download content from any URL."""
73 if content_type == ContentType.TEXT:
74 # For generic sources, we can only extract text from PDF
75 pdf_content = self._download_pdf(url)
76 if pdf_content: 76 ↛ 80line 76 didn't jump to line 80 because the condition on line 76 was always true
77 text = self.extract_text_from_pdf(pdf_content)
78 if text: 78 ↛ 80line 78 didn't jump to line 80 because the condition on line 78 was always true
79 return text.encode("utf-8")
80 return None
81 # Try to download as PDF
82 return self._download_pdf(url)
84 def download_with_result(
85 self, url: str, content_type: ContentType = ContentType.PDF
86 ) -> DownloadResult:
87 """Download content and return detailed result with skip reason."""
88 if content_type == ContentType.TEXT:
89 # For generic sources, we can only extract text from PDF
90 pdf_content = self._download_pdf(url)
91 if pdf_content:
92 text = self.extract_text_from_pdf(pdf_content)
93 if text:
94 return DownloadResult(
95 content=text.encode("utf-8"), is_success=True
96 )
97 return DownloadResult(
98 skip_reason="PDF downloaded but text extraction failed"
99 )
100 return DownloadResult(skip_reason="Could not download PDF from URL")
101 # Try to download as PDF
102 logger.info(
103 f"Attempting generic download from {redact_url_for_log(url)}"
104 )
106 # Try direct download
107 pdf_content = super()._download_pdf(url)
109 if pdf_content:
110 logger.info(
111 f"Successfully downloaded PDF from {redact_url_for_log(url)}"
112 )
113 return DownloadResult(content=pdf_content, is_success=True)
115 # If the URL doesn't end with .pdf, try adding it
116 pdf_url = _with_pdf_suffix(url)
117 if pdf_url is not None:
118 logger.debug(
119 f"Trying with .pdf extension: {redact_url_for_log(pdf_url)}"
120 )
121 pdf_content = super()._download_pdf(pdf_url)
122 else:
123 pdf_content = None
125 if pdf_content:
126 logger.info(
127 f"Successfully downloaded PDF from {redact_url_for_log(pdf_url)}"
128 )
129 return DownloadResult(content=pdf_content, is_success=True)
131 # Diagnostic request: determine WHY the download failed.
132 #
133 # IMPORTANT: stream=True is intentional here. DO NOT remove it.
134 # This block only inspects response.status_code and headers
135 # to determine why a download failed (404, 403, paywall, etc.).
136 # Without stream=True, the full response body would be downloaded
137 # into memory. Since GenericDownloader.can_handle() returns True
138 # for ALL URLs, this could mean downloading multi-GB files just
139 # to check a status code.
140 #
141 # The context manager (with ... as response) ensures the streamed
142 # connection is properly closed on all code paths, preventing
143 # file descriptor leaks (each unclosed stream=True response
144 # holds an open socket FD).
145 try:
146 with self.session.get(
147 url, timeout=5, allow_redirects=True, stream=True
148 ) as response:
149 # Check status code
150 if response.status_code == 200:
151 # HTML identifies a format mismatch, not a paywall.
152 response_content_type = response.headers.get(
153 "content-type", ""
154 ).lower()
155 if "text/html" in response_content_type:
156 return DownloadResult(
157 skip_reason=HTML_NOT_PDF_REASON,
158 status_code=response.status_code,
159 )
160 return DownloadResult(
161 skip_reason=f"Unexpected content type: {response_content_type} - expected PDF",
162 status_code=response.status_code,
163 )
164 if response.status_code == 404:
165 return DownloadResult(
166 skip_reason="Article not found (404) - may have been removed or URL is incorrect",
167 status_code=404,
168 )
169 if response.status_code == 403:
170 return DownloadResult(
171 skip_reason="Access denied (403) - article requires subscription or special permissions",
172 status_code=403,
173 )
174 if response.status_code == 401:
175 return DownloadResult(
176 skip_reason="Authentication required - please login to access this article",
177 status_code=401,
178 )
179 if response.status_code >= 500:
180 return DownloadResult(
181 skip_reason=f"Server error ({response.status_code}) - website is experiencing technical issues",
182 status_code=response.status_code,
183 )
184 return DownloadResult(
185 skip_reason=f"Unable to access article - server returned error code {response.status_code}",
186 status_code=response.status_code,
187 )
188 except requests.exceptions.Timeout:
189 return DownloadResult(
190 skip_reason="Connection timed out - server took too long to respond"
191 )
192 except requests.exceptions.ConnectionError:
193 return DownloadResult(
194 skip_reason="Could not connect to server - website may be down"
195 )
196 except requests.RequestException as e:
197 logger.opt(exception=False).warning(
198 "Unexpected error checking URL: {} ({})",
199 redact_url_for_log(url),
200 type(e).__name__,
201 )
202 return DownloadResult(
203 skip_reason="Network error - could not reach the website"
204 )
206 def _download_pdf(
207 self, url: str, headers: Optional[Dict[str, str]] = None
208 ) -> Optional[bytes]:
209 """Attempt to download PDF from URL."""
210 logger.info(
211 f"Attempting generic download from {redact_url_for_log(url)}"
212 )
214 # Try direct download
215 pdf_content = super()._download_pdf(url)
217 if pdf_content:
218 logger.info(
219 f"Successfully downloaded PDF from {redact_url_for_log(url)}"
220 )
221 return pdf_content
223 # If the URL doesn't end with .pdf, try adding it
224 pdf_url = _with_pdf_suffix(url)
225 if pdf_url is not None:
226 logger.debug(
227 f"Trying with .pdf extension: {redact_url_for_log(pdf_url)}"
228 )
229 pdf_content = super()._download_pdf(pdf_url)
230 else:
231 pdf_content = None
233 if pdf_content:
234 logger.info(
235 f"Successfully downloaded PDF from {redact_url_for_log(pdf_url)}"
236 )
237 return pdf_content
239 logger.warning(f"Failed to download PDF from {redact_url_for_log(url)}")
240 return None