Coverage for src/local_deep_research/research_library/downloaders/html.py: 97%
296 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2HTML Content Downloader for web pages.
4Downloads and extracts clean text content from HTML web pages.
5Extraction is handled by the shared pipeline in extraction/pipeline.py.
6"""
8from typing import Any, Callable, Dict, Optional
9from urllib.parse import urlparse
10from loguru import logger
11from bs4 import BeautifulSoup
12from requests.exceptions import ConnectionError as RequestsConnectionError
13from requests.exceptions import Timeout as RequestsTimeout
14from requests.exceptions import HTTPError as RequestsHTTPError
16from .base import (
17 _DOCUMENT_ABSENT_STATUSES,
18 BaseDownloader,
19 ContentType,
20 DownloadResult,
21 rate_limit_authority,
22)
23from .extraction.pipeline import extract_content_with_metadata
24from ...constants import BROWSER_USER_AGENT
25from ...security.client_safe_errors import (
26 CLIENT_SAFE_DOWNLOAD_MESSAGES,
27 client_safe_download_message,
28)
29from ...security.ssrf_validator import redact_url_for_log
30from ...security.egress.policy import PolicyDeniedError
31from ...utilities.pdf_extraction_limits import MAX_PDF_EXTRACTION_PAGES
32from ...utilities.resource_utils import safe_close
35# Single source of truth for caller-safe tokens lives in
36# ``security.client_safe_errors`` (PR #6564 review: this local copy drifted
37# from the services table — missing DataError/InterfaceError/
38# InvalidRequestError and the OSError errno branch). Aliases below preserve
39# the historic names; the mapping itself is shared so it cannot drift again.
40# Importing from ``security`` (not ``research_library.services``) avoids a
41# services <-> downloaders cycle. sanitize_error_for_client() is NOT a
42# substitute — it scrubs credential shapes only, not SQL/paths/URLs.
43_CLIENT_SAFE_HTML_MESSAGES = CLIENT_SAFE_DOWNLOAD_MESSAGES
46def _client_safe_html_message(exc: BaseException) -> str:
47 """Back-compat alias for :func:`client_safe_download_message`."""
48 return client_safe_download_message(exc)
51def _as_markdown_text(value: str) -> str:
52 """Collapse page-supplied metadata to a single line.
54 ``_format_extracted_content`` wraps the title in ``# `` and the
55 description in ``*``. Both are third-party strings -- for arXiv they now
56 reach stored text at character 0, ahead of the paper itself -- so a
57 newline in either would end the construct and hand the rest of the value
58 the document's own syntax. Collapsing whitespace keeps the value inside
59 the construct it was meant to fill.
61 Metacharacters are deliberately *not* escaped. What this produces is
62 stored as ``Document.text_content``, which is not markdown to every
63 consumer: the document viewer renders it as plain text in a
64 ``white-space: pre-wrap`` element (``document_text.html``), unified
65 search matches it by substring, and the RAG and notes services feed it
66 to embeddings and prompts verbatim. Backslashes written in here would
67 show up literally in all four and would break substring matches on the
68 characters escaped. Escaping belongs at the markdown sinks, and they
69 already do it -- the one that renders this text as markdown parses it
70 inline and passes the result through DOMPurify.
71 """
72 return " ".join(value.split())
75def _fetch_failure_reason(status_code: int) -> str:
76 """Factual HTTP-class token for ``html.fetch_failed`` log lines.
78 Names the status bucket only (absent / rate-limited / forbidden /
79 client / server error) — never the cause: a 403 may be a bot-wall
80 or a genuinely private document, and the log cannot tell.
81 """
82 if status_code in _DOCUMENT_ABSENT_STATUSES:
83 return "absent"
84 if status_code in (429, 503):
85 return "rate_limited"
86 if status_code in (401, 403):
87 return "forbidden"
88 if status_code == 400:
89 return "bad_request"
90 if 500 <= status_code <= 599:
91 return "server_error"
92 if 400 <= status_code <= 499:
93 return "client_error"
94 return "unexpected_status"
97class _TerminalTransportError(Exception):
98 """Internal: terminal transport failure with attempt context.
100 Raised by ``_fetch_response_with_retry`` when the last attempt fails
101 with a transport error, so the failure handling can record the real
102 attempt count instead of a hardcoded ``retry_count=1``. Never
103 escapes ``HTMLDownloader``.
104 """
106 def __init__(self, exc: BaseException, wait_time: float, attempt: int):
107 super().__init__(exc)
108 self.exc = exc
109 self.wait_time = wait_time
110 self.attempt = attempt
113class HTMLDownloader(BaseDownloader):
114 """Downloader for HTML web pages - extracts clean text content."""
116 # Bound on the PDF recovery path (see _extract_stashed_pdf_text). The
117 # stash is reachable from every HTML-classified URL, so the fetched
118 # bytes stay capped small (MAX_RECOVERED_PDF_BYTES); the page budget
119 # aliases the central MAX_PDF_EXTRACTION_PAGES ceiling so research
120 # recovery and dedicated PDF paths truncate identically (one source
121 # of truth in utilities.pdf_extraction_limits).
122 MAX_RECOVERED_PDF_BYTES = 10 * 1024 * 1024
123 MAX_RECOVERED_PDF_PAGES = MAX_PDF_EXTRACTION_PAGES
125 def __init__(
126 self,
127 timeout: int = 30,
128 language: str = "English",
129 **kwargs,
130 ):
131 super().__init__(timeout)
132 self.session.headers.update({"User-Agent": BROWSER_USER_AGENT})
133 self.language = language
135 def can_handle(self, url: str) -> bool:
136 """
137 Check if this downloader can handle the given URL.
139 Returns True for any HTTP/HTTPS URL (fallback downloader for web content).
140 """
141 try:
142 parsed = urlparse(url)
143 return parsed.scheme in ("http", "https")
144 except Exception:
145 return False
147 def download(
148 self, url: str, content_type: ContentType = ContentType.TEXT
149 ) -> Optional[bytes]:
150 """
151 Download and extract text content from HTML page.
153 Args:
154 url: The URL to download
155 content_type: Type of content (TEXT for HTML extraction)
157 Returns:
158 Extracted text as UTF-8 bytes, or None if failed
159 """
160 if content_type == ContentType.PDF:
161 logger.warning(
162 "html.download_pdf_unsupported url={url}",
163 url=redact_url_for_log(url),
164 )
165 return None
167 try:
168 html_content = self._fetch_html(url)
169 if not html_content:
170 kind, payload, final_url = self._consume_recovery(url)
171 if kind == "pdf" and isinstance(payload, bytes):
172 if len(payload) > self.MAX_RECOVERED_PDF_BYTES:
173 logger.info(
174 "html.pdf_recovery_skipped_oversize url={url}",
175 url=redact_url_for_log(url),
176 )
177 return None
178 pdf_text = self._extract_stashed_pdf_text(
179 payload, final_url or url, url
180 )
181 if pdf_text:
182 return pdf_text.encode("utf-8")
183 elif kind == "text" and isinstance(payload, str):
184 if payload.strip(): 184 ↛ 188line 184 didn't jump to line 188 because the condition on line 184 was always true
185 return self._format_recovered_text(
186 payload, final_url or url
187 ).encode("utf-8")
188 return None
190 extracted = self._extract_content(html_content, url)
191 if extracted:
192 text = self._format_extracted_content(extracted)
193 return text.encode("utf-8")
195 return None
197 except PolicyDeniedError:
198 raise
199 except Exception as e:
200 # Log the exception class only (zero-risk: the token already
201 # exposes it to the client). Never log str(e) or the traceback:
202 # either can re-embed the credentialed URL / SQL text / paths.
203 logger.opt(exception=False).error(
204 "html.download_failed url={url} exc_class={exc_class}",
205 url=redact_url_for_log(url),
206 exc_class=type(e).__name__,
207 )
208 return None
210 def download_with_result(
211 self,
212 url: str,
213 content_type: ContentType = ContentType.TEXT,
214 *,
215 on_transport_error: Optional[Callable[[BaseException], None]] = None,
216 ) -> DownloadResult:
217 """Download content, optionally reporting the typed terminal failure.
219 The callback receives only transport failures, before their details
220 are reduced to a client-safe skip reason. Extraction failures and
221 missing documents (404/410) remain per-document results.
222 """
223 if content_type == ContentType.PDF:
224 return DownloadResult(
225 skip_reason="HTML downloader does not support PDF downloads"
226 )
228 try:
229 if on_transport_error is None:
230 html_content = self._fetch_html(url)
231 else:
232 html_content, _ = self._fetch_html_with_final_url(
233 url, on_transport_error=on_transport_error
234 )
235 if not html_content:
236 kind, payload, final_url = self._consume_recovery(url)
237 if kind == "pdf" and isinstance(payload, bytes):
238 if len(payload) > self.MAX_RECOVERED_PDF_BYTES:
239 return DownloadResult(
240 skip_reason=(
241 f"PDF served at HTML URL exceeds "
242 f"{self.MAX_RECOVERED_PDF_BYTES // (1024 * 1024)} MB "
243 f"recovery limit, skipping text extraction"
244 )
245 )
246 pdf_text = self._extract_stashed_pdf_text(
247 payload, final_url or url, url
248 )
249 if pdf_text:
250 return DownloadResult(
251 content=pdf_text.encode("utf-8"),
252 is_success=True,
253 )
254 return DownloadResult(
255 skip_reason=(
256 "PDF served at HTML URL had no extractable text"
257 )
258 )
259 if kind == "text" and isinstance(payload, str):
260 if payload.strip(): 260 ↛ 267line 260 didn't jump to line 267 because the condition on line 260 was always true
261 return DownloadResult(
262 content=self._format_recovered_text(
263 payload, final_url or url
264 ).encode("utf-8"),
265 is_success=True,
266 )
267 return DownloadResult(
268 skip_reason="Text served at HTML URL was empty"
269 )
270 return DownloadResult(
271 skip_reason="Failed to fetch HTML content from URL"
272 )
274 extracted = self._extract_content(html_content, url)
275 if not extracted:
276 return DownloadResult(
277 skip_reason="Could not extract meaningful content from page"
278 )
280 text = self._format_extracted_content(extracted)
281 if not text.strip():
282 return DownloadResult(skip_reason="Extracted content is empty")
284 return DownloadResult(
285 content=text.encode("utf-8"),
286 is_success=True,
287 )
289 except PolicyDeniedError:
290 raise
291 except Exception as e:
292 logger.opt(exception=False).error(
293 "html.download_result_failed url={url} exc_class={exc_class}",
294 url=redact_url_for_log(url),
295 exc_class=type(e).__name__,
296 )
297 # skip_reason propagates to the browser via the download SSE
298 # stream; the exception text can carry SQL text, target URLs, or
299 # file paths — return a fixed token only (never str(e); only the
300 # class name is logged above, as a traceback could re-embed the
301 # credentialed URL).
302 return DownloadResult(skip_reason=_client_safe_html_message(e))
304 def _fetch_html(
305 self,
306 url: str,
307 *,
308 on_rate_limited: Optional[Callable[[], None]] = None,
309 ) -> Optional[str]:
310 """Fetch raw HTML content from URL."""
311 return self._fetch_html_with_final_url(
312 url, on_rate_limited=on_rate_limited
313 )[0]
315 def _fetch_html_with_final_url(
316 self,
317 url: str,
318 *,
319 on_transport_failure: Optional[Callable[[], None]] = None,
320 on_transport_error: Optional[Callable[[BaseException], None]] = None,
321 on_rate_limited: Optional[Callable[[], None]] = None,
322 max_attempts: int = 3,
323 ) -> tuple[Optional[str], Optional[str]]:
324 """Fetch raw HTML together with the URL that finally served it.
326 Redirects are followed, so the final URL can differ from ``url``.
327 Callers that require the response to stay on a specific path must
328 compare the returned URL themselves.
330 A response that is not HTML is still a failure here (``(None, None)``)
331 so strict callers (arXiv HTML rendition check, metadata) keep their
332 contract. Recoverable responses -- a PDF, or plain text / a feed --
333 have their payload stashed for ``download()``/
334 ``download_with_result()`` (see ``_consume_recovery``): the URL
335 classifier only sees the URL string, so publisher/DOI links without
336 a ``.pdf`` path routinely land here serving ``application/pdf``,
337 and plain-text / feed URLs (READMEs, ``.txt`` preprints, RSS/Atom)
338 likewise arrive typed as something other than HTML. Discarding
339 those bytes turns a recoverable source into a warning plus a lost
340 citation.
342 Transient failures -- HTTP 429/503 and ``requests`` timeouts /
343 connection errors -- are retried up to ``max_attempts`` with an
344 adaptive wait before every attempt (mirroring
345 ``BaseDownloader._download_pdf``); every other status or error
346 fails fast exactly as before. ``on_transport_failure`` fires at
347 most once, only on the terminal failure, preserving its
348 circuit-breaker contract (see below).
350 ``on_transport_failure`` is called once when the fetch fails
351 without the host answering: an exception (timeout, connection
352 error, a body that aborts mid-read, the decoded-body cap) or a
353 non-200 status other than 404 and 410, which answer "no such
354 page". A 200 that is not HTML is an answer too and does not call
355 it, but only once its body has been read in full: a 200 whose
356 body aborts mid-read never got to answer, and is reported as the
357 transport failure it is. When given, the failure is logged
358 as a warning rather than an error, since the caller handles it.
360 ``on_rate_limited`` is called once when the fetch gives up on
361 HTTP 429/503 after exhausting ``max_attempts`` (mirroring
362 ``BaseDownloader._download_pdf``): a chain owner uses it to stop
363 sending further requests to that host. Only the caller can stop
364 the later legs of the fallback chain: the probe GET applies no
365 rate limiting, and the browser leg backs off only on its own
366 tracker buckets, which never see the 429/503 outcomes this fetch
367 records.
368 """
369 # Fresh per-fetch recovery state. Cleared on entry so a stale stash
370 # from an earlier fetch can never leak into this one; consumption
371 # additionally requires the stashed request URL to match (see
372 # _consume_recovery). The stash fields are unsynchronized instance
373 # state: downloader instances are used sequentially (e.g. per-request
374 # construction in fetch_and_extract, or sequential consumption of
375 # one shared instance in batch_fetch_and_extract) — cross-thread
376 # sharing of one instance is unsupported, and the URL-match guard
377 # does not make it safe (see _consume_recovery).
378 self._recovery_kind: Optional[str] = None
379 self._recovery_payload: Optional[bytes | str] = None
380 self._recovery_url: Optional[str] = None
381 self._recovery_request_url: Optional[str] = url
382 logger.debug(
383 "html.fetch_started url={url}",
384 url=redact_url_for_log(url),
385 )
386 engine_type = f"html_download_{rate_limit_authority(url)}"
387 max_attempts = max(1, int(max_attempts))
389 def _terminal_failure(
390 exc: BaseException, failed_wait_time: float, failed_attempt: int
391 ) -> tuple[None, None]:
392 # A fetch that raised must leave no recovery state behind, even
393 # if it raised after the stash was populated: the payload belongs
394 # to a response the caller never received in full.
395 self._recovery_kind = None
396 self._recovery_payload = None
397 self._recovery_url = None
398 if on_transport_failure is not None:
399 # The caller handles the failure, so it is not an error
400 # here (ERROR records reach the user's browser).
401 logger.warning(
402 "html.fetch_error url={url} exc_class={exc_class}",
403 url=redact_url_for_log(url),
404 exc_class=type(exc).__name__,
405 )
406 else:
407 logger.opt(exception=False).error(
408 "html.fetch_error url={url} exc_class={exc_class}",
409 url=redact_url_for_log(url),
410 exc_class=type(exc).__name__,
411 )
412 self.rate_tracker.record_outcome(
413 engine_type=engine_type,
414 wait_time=failed_wait_time,
415 success=False,
416 retry_count=failed_attempt,
417 error_type=type(exc).__name__,
418 )
419 if on_transport_failure is not None:
420 on_transport_failure()
421 if on_transport_error is not None:
422 on_transport_error(exc)
423 return None, None
425 wait_time = 0.0
426 attempt = 1
427 try:
428 response, wait_time, attempt = self._fetch_response_with_retry(
429 url, engine_type, max_attempts
430 )
432 try:
433 if response.status_code == 200:
434 content_type = response.headers.get(
435 "content-type", ""
436 ).lower()
437 if (
438 "text/html" in content_type
439 or "application/xhtml" in content_type
440 ):
441 self._ensure_decoded_body_cap(response)
442 # Read (capped) before recording the outcome, so a
443 # body that aborts mid-read is not also counted as
444 # a success.
445 text = response.text
446 self.rate_tracker.record_outcome(
447 engine_type=engine_type,
448 wait_time=wait_time,
449 success=True,
450 retry_count=attempt,
451 search_result_count=1,
452 )
453 logger.debug(
454 "html.fetch_succeeded url={url}",
455 url=redact_url_for_log(url),
456 )
457 return text, response.url
458 # Non-HTML 200 is an answer, not a transport failure: do
459 # not call on_transport_failure. Bound the body before any
460 # recovery read, including the magic-byte check below.
461 # _ensure_decoded_body_cap adds the decoded-byte guard for
462 # a small-Content-Length gzip body on top of the one
463 # SafeSession.send installs; the read that follows is
464 # then capped at MAX_RESPONSE_SIZE.
465 self._ensure_decoded_body_cap(response)
466 # Read the bounded body HERE, inside this method's outer
467 # try, instead of letting _is_pdf_content() do it. That
468 # helper swallows every exception raised while reading
469 # response.content and answers "not a PDF", so a body
470 # that aborts mid-read (connection reset, truncated
471 # chunked body) falls through to the unsupported-content
472 # return below: an answer the host never gave, recorded
473 # as neither on_transport_failure nor a failed outcome.
474 # The arXiv caller then reads the attempt as NOT_FETCHED
475 # (a refundable fetch) instead of FETCH_FAILED and keeps
476 # sending requests to a failing host. Reading here routes
477 # the failure to the outer except, which records it
478 # exactly once, and the same bytes are reused for the
479 # sniff and the stash below, so this costs no second
480 # read. Mirrors the PDF path (BaseDownloader.
481 # _download_pdf), which reads its body the same way.
482 body = response.content
483 if self._is_pdf_content(response, content=body):
484 # HTML-classified URL serving a PDF: not a
485 # warning-worthy failure. Stash the already-downloaded
486 # (bounded) bytes so download()/download_with_result()
487 # can extract text with no second request (one-GET
488 # recovery contract).
489 logger.info(
490 "html.fetch_pdf_recovery url={url}",
491 url=redact_url_for_log(url),
492 )
493 self._recovery_kind = "pdf"
494 # The bytes read above, reused: no second request and
495 # no second read of the body.
496 self._recovery_payload = body
497 self._recovery_url = (
498 getattr(response, "url", None) or url
499 )
500 return None, None
501 if self._is_recoverable_text_content(content_type):
502 # Plain text / feed served where HTML was expected
503 # (READMEs, .txt preprints, RSS/Atom): usable as source
504 # text directly, no extraction step needed.
505 logger.info(
506 "html.fetch_text_recovery url={url}",
507 url=redact_url_for_log(url),
508 )
509 self._recovery_kind = "text"
510 self._recovery_payload = getattr(response, "text", "")
511 self._recovery_url = (
512 getattr(response, "url", None) or url
513 )
514 return None, None
515 logger.warning(
516 "html.fetch_unexpected_content_type url={url} status={status}",
517 url=redact_url_for_log(url),
518 status=response.status_code,
519 )
520 return None, None
521 status = response.status_code
522 # NOTE: intermediate 429/503s never reach here --
523 # _fetch_response_with_retry handles (and closes) those
524 # before returning. A 429/503 seen here exhausted
525 # max_attempts and fails terminally below.
526 logger.warning(
527 "html.fetch_failed url={url} status={status} "
528 "reason={reason} attempts={attempts}",
529 url=redact_url_for_log(url),
530 status=status,
531 reason=_fetch_failure_reason(status),
532 attempts=attempt,
533 )
534 self.rate_tracker.record_outcome(
535 engine_type=engine_type,
536 wait_time=wait_time,
537 success=False,
538 retry_count=attempt,
539 error_type=f"HTTP_{status}",
540 )
541 if (
542 on_transport_failure is not None
543 and status not in _DOCUMENT_ABSENT_STATUSES
544 ):
545 on_transport_failure()
546 if (
547 on_transport_error is not None
548 and status not in _DOCUMENT_ABSENT_STATUSES
549 ):
550 on_transport_error(RequestsHTTPError(response=response))
551 if status in (429, 503) and on_rate_limited is not None:
552 # Fires exactly once: this branch is terminal (the
553 # retried 429/503s never reach it — see the NOTE above).
554 # Mirrors ``BaseDownloader._download_pdf``: the caller
555 # stops sending further requests to this host.
556 on_rate_limited()
557 return None, None
558 finally:
559 # stream=True leaves the connection unreleased until the
560 # body is consumed or the response is closed.
561 safe_close(response, "HTML download response")
563 except PolicyDeniedError:
564 self._recovery_kind = None
565 self._recovery_payload = None
566 self._recovery_url = None
567 raise
568 except _TerminalTransportError as terminal:
569 return _terminal_failure(
570 terminal.exc, terminal.wait_time, terminal.attempt
571 )
572 except Exception as e:
573 return _terminal_failure(e, wait_time, attempt)
575 def _fetch_response_with_retry(
576 self, url: str, engine_type: str, max_attempts: int
577 ) -> tuple[Any, float, int]:
578 """GET with retry on transient failures; terminal errors raise.
580 Retries HTTP 429/503 and ``requests`` timeouts / connection
581 errors (mirroring ``BaseDownloader._download_pdf``), with an
582 adaptive wait before every attempt. Returns
583 ``(response, wait_time, attempt)``. A terminal transport failure
584 raises ``_TerminalTransportError`` carrying the attempt context;
585 any other exception from the GET propagates unchanged. A
586 retried 429/503 response is closed before continuing so pooled
587 connections are released.
588 """
589 for attempt in range(1, max_attempts + 1):
590 # Adaptive wait before every attempt (mirrors
591 # BaseDownloader._download_pdf): the tracker learns per host,
592 # and retrying into a rate-limited host without waiting would
593 # trip the next attempt immediately.
594 wait_time = self.rate_tracker.apply_rate_limit(engine_type)
595 try:
596 # stream=True keeps requests.Session.send from consuming the
597 # body before _ensure_decoded_body_cap can wrap the reads, so
598 # that guard also bounds the decoded read of response.text at
599 # MAX_RESPONSE_SIZE for a valid under-cap Content-Length (a
600 # second guard: SafeSession.send installs one on every response
601 # it accepts). This bounds the final response only: with
602 # allow_redirects=True, SafeSession discards each intermediate
603 # redirect body unread.
604 response = self.session.get(
605 url,
606 timeout=self.timeout,
607 allow_redirects=True,
608 stream=True,
609 )
610 except PolicyDeniedError:
611 raise
612 except (RequestsTimeout, RequestsConnectionError) as e:
613 if attempt >= max_attempts:
614 raise _TerminalTransportError(e, wait_time, attempt) from e
615 logger.warning(
616 "html.fetch_attempt url={url} exc_class={exc_class} "
617 "attempt={attempt}/{max_attempts}",
618 url=redact_url_for_log(url),
619 exc_class=type(e).__name__,
620 attempt=attempt,
621 max_attempts=max_attempts,
622 )
623 self.rate_tracker.record_outcome(
624 engine_type=engine_type,
625 wait_time=wait_time,
626 success=False,
627 retry_count=attempt,
628 error_type=type(e).__name__,
629 )
630 continue
631 except Exception as e:
632 raise _TerminalTransportError(e, wait_time, attempt) from e
633 status = response.status_code
634 if status in (429, 503) and attempt < max_attempts:
635 logger.warning(
636 "html.fetch_attempt url={url} status={status} "
637 "attempt={attempt}/{max_attempts}",
638 url=redact_url_for_log(url),
639 status=status,
640 attempt=attempt,
641 max_attempts=max_attempts,
642 )
643 self.rate_tracker.record_outcome(
644 engine_type=engine_type,
645 wait_time=wait_time,
646 success=False,
647 retry_count=attempt,
648 error_type=f"HTTP_{status}",
649 )
650 safe_close(response, "HTML download response")
651 continue
652 return response, wait_time, attempt
653 raise AssertionError( # pragma: no cover -- defensive
654 "_fetch_response_with_retry: loop always returns or raises"
655 )
657 @staticmethod
658 def _is_recoverable_text_content(content_type: str) -> bool:
659 """Report whether a non-HTML content type is usable as text directly.
661 ``text/*`` plus feed/syndication XML (RSS/Atom, ``application/xml``,
662 and other ``+xml`` types except SVG, which is markup, not prose).
663 Deliberately excludes ``application/json``: raw API dumps are large
664 and low-signal as research sources, so they keep failing loudly
665 instead of silently entering the corpus.
666 """
667 # Headers routinely carry parameters (``; charset=utf-8``). Strip
668 # them before matching, otherwise ``application/rss+xml;
669 # charset=utf-8`` matches neither the exact names nor ``+xml``.
670 # NOTE: local name intentionally avoids ``media_type`` so the
671 # stored-XSS census (``media_type\s*=``) keeps passing -- this
672 # value never reaches a served response.
673 base_type = content_type.split(";", 1)[0].strip().lower()
674 # SVG is markup, not prose -- reject in every family, including
675 # ``text/svg`` and ``text/svg+xml`` which would otherwise pass via
676 # the ``text/`` prefix below.
677 if "svg" in base_type:
678 return False
679 if base_type.startswith("text/"):
680 return True
681 if base_type in (
682 "application/xml",
683 "application/rss+xml",
684 "application/atom+xml",
685 ):
686 return True
687 return base_type.endswith("+xml")
689 def _consume_recovery(
690 self, url: str
691 ) -> tuple[Optional[str], Optional[bytes | str], Optional[str]]:
692 """Take stashed recovery payload when it belongs to this request URL.
694 Returns ``(kind, payload, final_url)`` with kind ``"pdf"`` (bytes)
695 or ``"text"`` (str), or ``(None, None, None)``. Single-use: the
696 stash is cleared on a URL match so each fetch consumes at most its
697 own bytes. A mismatch (stale stash from an earlier fetch) is left
698 alone and reported as absent. The stash fields are unsynchronized
699 instance state and downloader instances are used sequentially
700 (per-request construction, or sequential consumption of one shared
701 instance); sharing one instance across threads is unsupported. The
702 URL-match guard does not make such sharing safe: the request URL is
703 recorded when a fetch starts, not when the stash is written, and
704 the non-200 and unexpected-content-type returns do not clear the
705 stash. If one instance is shared anyway, a request can lose its own
706 recovery or receive another request's recovered content.
707 """
708 if getattr(self, "_recovery_payload", None) is None:
709 return None, None, None
710 if getattr(self, "_recovery_request_url", None) != url: 710 ↛ 711line 710 didn't jump to line 711 because the condition on line 710 was never true
711 return None, None, None
712 kind = getattr(self, "_recovery_kind", None)
713 payload = self._recovery_payload
714 final_url = getattr(self, "_recovery_url", None) or url
715 self._recovery_payload = None
716 self._recovery_kind = None
717 return kind, payload, final_url
719 def _extract_stashed_pdf_text(
720 self, pdf_bytes: bytes, pdf_url: str, url: str
721 ) -> Optional[str]:
722 """Extract and format text from stashed PDF bytes."""
723 # Defense in depth: callers check the byte cap first for a specific
724 # skip reason, but never parse an oversize stash even if they don't.
725 if len(pdf_bytes) > self.MAX_RECOVERED_PDF_BYTES: 725 ↛ 726line 725 didn't jump to line 726 because the condition on line 725 was never true
726 logger.info(
727 "html.pdf_recovery_skipped_oversize url={url}",
728 url=redact_url_for_log(url),
729 )
730 return None
731 try:
732 text = self.extract_text_from_pdf(
733 pdf_bytes, max_pages=self.MAX_RECOVERED_PDF_PAGES
734 )
735 except Exception:
736 logger.opt(exception=False).error(
737 "html.pdf_recovery_extract_failed url={url}",
738 url=redact_url_for_log(url),
739 )
740 return None
741 if not text or not text.strip():
742 logger.debug(
743 "html.pdf_recovery_empty url={url}",
744 url=redact_url_for_log(url),
745 )
746 return None
747 return self._format_recovered_text(text, pdf_url)
749 @staticmethod
750 def _format_recovered_text(text: str, url: str) -> str:
751 """Format recovered non-HTML text like content (Source line + body)."""
752 return f"Source: {url}\n\n{text.strip()}"
754 def _extract_content(self, html: str, url: str) -> Optional[Dict[str, Any]]:
755 """Extract clean content and metadata from HTML.
757 Delegates to the shared extraction pipeline which handles
758 trafilatura, readability, justext, and metadata enrichment.
759 """
760 try:
761 result = extract_content_with_metadata(html, language=self.language)
762 if not result:
763 return None
765 title = result.get("title")
766 content = result["content"]
768 logger.debug(
769 "html.extract_succeeded url={url} content_length={content_length}",
770 url=redact_url_for_log(url),
771 content_length=len(content),
772 )
773 return {
774 "title": title,
775 "description": result.get("description"),
776 "content": content,
777 "url": url,
778 }
780 except Exception as e:
781 logger.opt(exception=False).error(
782 "html.extract_failed url={url} exc_class={exc_class}",
783 url=redact_url_for_log(url),
784 exc_class=type(e).__name__,
785 )
786 return None
788 def _format_extracted_content(self, extracted: Dict[str, Any]) -> str:
789 """Format extracted content as readable text."""
790 parts = []
792 if extracted.get("title"):
793 parts.append(f"# {_as_markdown_text(str(extracted['title']))}")
794 parts.append("")
796 if extracted.get("description"):
797 parts.append(
798 f"*{_as_markdown_text(str(extracted['description']))}*"
799 )
800 parts.append("")
802 if extracted.get("url"):
803 parts.append(f"Source: {extracted['url']}")
804 parts.append("")
806 if extracted.get("content"):
807 parts.append(extracted["content"])
809 return "\n".join(parts)
811 def get_metadata(self, url: str) -> Dict[str, Any]:
812 """Get metadata about the page."""
813 html_content = self._fetch_html(url)
814 if not html_content:
815 return {}
817 try:
818 soup = BeautifulSoup(html_content, "html.parser")
820 metadata = {"url": url}
822 if soup.title and soup.title.string: 822 ↛ 825line 822 didn't jump to line 825 because the condition on line 822 was always true
823 metadata["title"] = soup.title.string.strip()
825 meta_desc = soup.find("meta", attrs={"name": "description"})
826 if meta_desc and meta_desc.get("content"):
827 metadata["description"] = str(meta_desc["content"])
829 author = soup.find("meta", attrs={"name": "author"})
830 if author and author.get("content"):
831 metadata["author"] = str(author["content"])
833 for prop in ["article:published_time", "datePublished"]:
834 date_tag = soup.find("meta", property=prop)
835 if date_tag and date_tag.get("content"):
836 metadata["published_date"] = str(date_tag["content"])
837 break
839 return metadata
841 except Exception as e:
842 logger.opt(exception=False).error(
843 "html.metadata_extract_failed url={url} exc_class={exc_class}",
844 url=redact_url_for_log(url),
845 exc_class=type(e).__name__,
846 )
847 return {"url": url}