Coverage for src/local_deep_research/research_library/downloaders/playwright_html.py: 90%
552 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2HTML Downloader with JavaScript rendering support.
4Uses Crawl4AI (default) or plain Playwright for JS-rendered pages.
5Crawl4AI adds: robots.txt checking, shadow DOM flattening, iframe
6inlining, smart scrolling for lazy-loaded content, and caching.
7Falls back to plain Playwright if Crawl4AI is not installed.
9No stealth/anti-detection features are used — the browser identifies
10honestly via BROWSER_USER_AGENT and respects robots.txt.
11"""
13import asyncio
14import functools
15from datetime import datetime, timezone
16from email.utils import parsedate_to_datetime
17from typing import Optional
18from urllib.parse import urljoin, urlparse
20from loguru import logger
22from .base import rate_limit_authority
23from .html import HTMLDownloader
24from ...constants import BROWSER_USER_AGENT
25from ...security import ssrf_validator
26from ...security.egress.policy import Decision, EgressScope, PolicyDeniedError
27from ...security.ssrf_validator import redact_url_for_log
28from ...security.safe_requests import (
29 _MAX_REDIRECTS,
30 _REDIRECT_STATUS_CODES,
31 _resolve_redirect_method,
32)
34# Heavy subresource types dropped when ``block_resources`` is set, mirroring
35# the previous extension-glob resource route but keyed on Playwright's
36# resource_type (robust to query strings / extensionless URLs).
37_BLOCKED_RESOURCE_TYPES = frozenset({"image", "media", "font", "stylesheet"})
39# Non-http(s) URL schemes the browser may safely handle itself (they never
40# touch the network, so they are not SSRF / local-resource vectors): the route
41# guard lets ONLY these fall through via ``route.fallback()``. EVERY other
42# non-http(s) scheme — ``file:`` (local file read), ``chrome:``/``chrome-
43# extension:``, ``gopher:``, ``ftp:``, ``ws(s):`` handled elsewhere, etc. — is
44# aborted fail-closed rather than blanket-allowed, so a redirect (or a page
45# resource) cannot pivot the browser to a local file or an internal handler.
46_FALLTHROUGH_NON_HTTP_SCHEMES = frozenset({"data", "blob", "about"})
48# Neutralise Service Workers on browser paths that cannot set
49# service_workers="block" at context creation (Crawl4AI's BrowserConfig
50# exposes no such option). Requests made BY a Service Worker are not
51# intercepted by page/context route handlers, so a SW could otherwise
52# fetch an internal/metadata host unvalidated. Preventing registration
53# forces all page traffic back through the guarded normal network path.
54# Injected as an init script so it runs before any page script.
55_DISABLE_SERVICE_WORKERS_JS = """
56try {
57 if (window.navigator && window.navigator.serviceWorker) {
58 const reject = () =>
59 Promise.reject(new Error('service workers are disabled'));
60 try { window.navigator.serviceWorker.register = reject; } catch (e) {}
61 try {
62 Object.defineProperty(window.navigator, 'serviceWorker', {
63 configurable: true,
64 get() { return undefined; },
65 });
66 } catch (e) {}
67 }
68} catch (e) {}
69"""
72# SSRF hardening: WebRTC (RTCPeerConnection ICE / STUN / TURN) opens raw
73# UDP/TCP sockets to an arbitrary host:port through Chromium's WebRTC stack —
74# a data path that neither page.route() (Fetch) nor route_web_socket()
75# instruments, so it bypasses ssrf_validator entirely. Text extraction never
76# needs WebRTC, so neutralize the constructors before any page script runs
77# (same fail-closed reasoning as the service-worker block).
78_DISABLE_WEBRTC_JS = """
79(() => {
80 const deny = function () {
81 throw new DOMException('WebRTC is disabled', 'NotSupportedError');
82 };
83 for (const name of ['RTCPeerConnection', 'webkitRTCPeerConnection',
84 'mozRTCPeerConnection', 'RTCDataChannel']) {
85 try {
86 Object.defineProperty(window, name, {
87 configurable: false, enumerable: false, get() { return deny; },
88 });
89 } catch (e) {}
90 }
91})();
92"""
95# Signals that a page is a JS-rendered SPA and needs browser rendering
96SPA_SIGNALS = [
97 'id="root"',
98 'id="app"',
99 'id="__next"',
100 "__NEXT_DATA__",
101 "data-reactroot",
102 'ng-version="',
103 "<noscript>You need to enable JavaScript",
104 "<noscript>Please enable JavaScript",
105 "window.__INITIAL_STATE__",
106]
109def _browser_transport_allowed(session) -> bool:
110 """Native browser DNS cannot enforce a restricted run's pinned scope."""
111 context = getattr(session, "egress_context", None)
112 return context is None or getattr(context, "scope", None) in (
113 EgressScope.BOTH,
114 EgressScope.UNPROTECTED,
115 )
118def _cookie_domain_matches(host: str, cookie_domain: str) -> bool:
119 """Whether a ``Set-Cookie`` ``Domain=`` attribute may be honored for a
120 response served by ``host``.
122 Mirrors RFC 6265 §5.1.3 domain-matching (and real-browser behavior): the
123 host must equal the cookie domain or be a subdomain of it. A leading dot
124 on the attribute is ignored. Returns False for a Domain that points at a
125 DIFFERENT or unrelated registrable domain — a cookie a browser drops —
126 so the caller can fall back to host-scoping instead of letting a redirect
127 hop plant a cookie on someone else's domain (session fixation).
129 Note: this does NOT consult the Public Suffix List, so a suffix like
130 ``Domain=co.uk`` set by ``a.co.uk`` is honored here and could be re-sent
131 to a sibling host later in the SAME redirect chain. The residual is very
132 narrow — the browser context is ephemeral (one fetch) and Chromium
133 enforces the PSL on its own jar — and it is strictly better than the
134 pre-fix behavior, which forwarded all cookies cross-domain. Wiring in a
135 PSL check would tighten it further (optional follow-up).
136 """
137 if not host or not cookie_domain: 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true
138 return False
139 host = host.lower().rstrip(".")
140 cookie_domain = cookie_domain.lower().lstrip(".").rstrip(".")
141 if not cookie_domain: 141 ↛ 142line 141 didn't jump to line 142 because the condition on line 141 was never true
142 return False
143 return host == cookie_domain or host.endswith("." + cookie_domain)
146def _same_registrable_site(host_a: str, host_b: str) -> bool:
147 """Whether two request hosts belong to the same domain family for the
148 purpose of forwarding credentialed request headers across a redirect hop.
150 Mirrors the host/subdomain relation ``_cookie_domain_matches`` already
151 uses (RFC 6265 §5.1.3 style): equal hosts, or one a subdomain of the
152 other, count as the SAME site; anything else is cross-site. Like that
153 helper it does NOT consult the Public Suffix List, so it is a deliberately
154 conservative approximation of "registrable domain" — consistent with the
155 rest of this guard, while Chromium still enforces its own PSL on the
156 browser jar. Used to decide when to drop ``Authorization`` /
157 ``Proxy-Authorization`` on a cross-site hop, the twin of the per-hop
158 ``Cookie`` re-scoping (a browser strips credentials cross-origin too).
160 Fail-CLOSED: a missing/empty host on either side returns False (treated as
161 cross-site), so credentialed headers are dropped rather than forwarded on
162 any ambiguity.
163 """
164 if not host_a or not host_b: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 return False
166 a = host_a.lower().rstrip(".")
167 b = host_b.lower().rstrip(".")
168 if not a or not b: 168 ↛ 169line 168 didn't jump to line 169 because the condition on line 168 was never true
169 return False
170 return a == b or a.endswith("." + b) or b.endswith("." + a)
173def _parse_one_set_cookie(raw: str):
174 """Parse a single ``Set-Cookie`` header value into ``(name, value, attrs)``.
176 A manual RFC 6265 §5.2 split rather than ``http.cookies.SimpleCookie``:
177 SimpleCookie silently DROPS cookies whose name is not a strict RFC token
178 (e.g. ``data[uid]``, which real browsers accept), so those cookies would
179 vanish from the re-applied jar. This split keeps them.
181 Returns ``None`` when the header has no ``name=value`` pair. ``attrs``
182 keys are lowercased; value-less flag attributes (``Secure``, ``HttpOnly``)
183 map to ``True``. Each ``Set-Cookie`` header carries exactly one cookie
184 (RFC 6265 §3), and the only comma that can appear inside a value belongs
185 to an ``Expires`` HTTP-date, which never contains a semicolon — so a
186 plain ``;`` split is safe.
187 """
188 if not raw: 188 ↛ 189line 188 didn't jump to line 189 because the condition on line 188 was never true
189 return None
190 parts = raw.split(";")
191 name, sep, value = parts[0].partition("=")
192 name = name.strip()
193 if not sep or not name:
194 return None
195 value = value.strip()
196 # Strip one layer of surrounding DQUOTEs (mirrors SimpleCookie).
197 if len(value) >= 2 and value[0] == '"' and value[-1] == '"': 197 ↛ 198line 197 didn't jump to line 198 because the condition on line 197 was never true
198 value = value[1:-1]
199 attrs = {}
200 for segment in parts[1:]:
201 akey, asep, aval = segment.partition("=")
202 akey = akey.strip().lower()
203 if not akey: 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true
204 continue
205 attrs[akey] = aval.strip() if asep else True
206 return name, value, attrs
209def _set_cookie_is_expired(attrs: dict) -> bool:
210 """Whether a parsed ``Set-Cookie`` is a deletion/expired directive that
211 must NOT be re-applied to the jar.
213 A ``Max-Age=0`` (or negative) or a past ``Expires`` is how a server
214 DELETES a cookie; re-adding it as a live cookie would resurrect a value
215 the server just cleared. ``Max-Age`` takes precedence over ``Expires``
216 (RFC 6265 §5.3). Fail-OPEN: an unparseable attribute is treated as NOT
217 expired, so a live cookie is never dropped by a date-parsing quirk.
218 """
219 max_age = attrs.get("max-age")
220 if isinstance(max_age, str) and max_age.strip():
221 try:
222 return int(max_age) <= 0
223 except ValueError:
224 pass # Unparseable Max-Age -> fall through to Expires.
225 expires = attrs.get("expires")
226 if isinstance(expires, str) and expires.strip():
227 try:
228 dt = parsedate_to_datetime(expires)
229 if dt is None: 229 ↛ 230line 229 didn't jump to line 230 because the condition on line 229 was never true
230 return False
231 if dt.tzinfo is None: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true
232 dt = dt.replace(tzinfo=timezone.utc)
233 return dt <= datetime.now(timezone.utc)
234 except Exception:
235 return False
236 return False
239def _run_async(coro, timeout: float = None):
240 """Run an async coroutine from synchronous code.
242 Handles the case where an event loop is already running
243 (e.g. inside Jupyter or an async framework) by creating
244 a new thread with its own loop.
246 Args:
247 coro: The coroutine to run.
248 timeout: Max seconds to wait for the result. Prevents
249 indefinite hangs if the coroutine's internal timeout fails.
250 """
251 try:
252 loop = asyncio.get_running_loop()
253 except RuntimeError:
254 loop = None
256 if loop is None:
257 return asyncio.run(coro)
259 # Already inside an event loop — run in a new thread
260 import concurrent.futures
262 with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
263 future = pool.submit(asyncio.run, coro)
264 return future.result(timeout=timeout)
267class PlaywrightHTMLDownloader(HTMLDownloader):
268 """HTML downloader with JS rendering via Crawl4AI or Playwright.
270 Default: Crawl4AI (robots.txt, shadow DOM, iframes, caching).
271 Fallback: plain Playwright if Crawl4AI is not installed.
273 No stealth or anti-detection features are used.
274 """
276 def __init__(
277 self,
278 timeout: int = 30,
279 language: str = "English",
280 wait_until: str = "networkidle",
281 block_resources: bool = True,
282 allow_private_ips: bool = False,
283 *,
284 block_link_local: bool = False,
285 **kwargs,
286 ):
287 super().__init__(timeout=timeout, language=language)
288 self.wait_until = wait_until
289 self.block_resources = block_resources
290 # SSRF policy for the browser route guards (initial navigation +
291 # every redirect hop + subresources). Defaults strict; relaxed to
292 # True only under the PRIVATE_ONLY egress scope so local lab hosts
293 # remain reachable. Cloud-metadata IPs stay blocked regardless
294 # (``ssrf_validator`` always rejects them). Mirrors the static
295 # SafeSession path's ``allow_private_ips`` handling.
296 self.allow_private_ips = allow_private_ips
297 # Link-local carve-out inside the private-IP grant (see
298 # AutoHTMLDownloader): both route guards validate every hop with it,
299 # so a redirect into the link-local range is refused on the JS route
300 # exactly as the static route refuses it. No effect without
301 # ``allow_private_ips``.
302 self.block_link_local = block_link_local
303 # The guarded robots.txt check (below) reuses this SafeSession, whose
304 # send() SSRF-validates every hop; keep its policy in lockstep with
305 # the browser guard so both honour the same egress scope.
306 if hasattr(self.session, "allow_private_ips"): 306 ↛ 308line 306 didn't jump to line 308 because the condition on line 306 was always true
307 self.session.allow_private_ips = allow_private_ips
308 if hasattr(self.session, "block_link_local"): 308 ↛ 311line 308 didn't jump to line 311 because the condition on line 308 was always true
309 self.session.block_link_local = block_link_local
310 # Plain Playwright fallback state
311 self._playwright = None
312 self._browser = None
314 # ------------------------------------------------------------------
315 # Per-request egress guard for the JS-render browser paths.
316 #
317 # WHY a manual redirect loop instead of "validate the request URL and
318 # continue": Playwright does NOT re-fire ``page.route`` handlers for the
319 # hops of a server-side redirect chain — the browser follows 3xx
320 # responses internally, below the interception layer. So a
321 # ``route.continue_()`` / ``route.fallback()`` on a public URL that
322 # 302-redirects to a cloud-metadata endpoint (link-local) or an RFC1918
323 # host would let the browser reach the internal target unchecked —
324 # exactly the gap the static ``SafeSession`` path already closes by
325 # validating every hop. To reach parity we follow the redirect chain
326 # ourselves with ``route.fetch(max_redirects=0)``, validate EACH hop's
327 # target before fetching it, and hand the browser only the final, safe
328 # response via ``route.fulfill``. The browser therefore never issues its
329 # own redirect-following request to an unvalidated destination.
330 # ------------------------------------------------------------------
332 def _playwright_route_guard(self, route) -> None:
333 """Egress guard for the plain-Playwright (sync) browser path.
335 Validates the initial navigation and every redirect hop against
336 ``ssrf_validator.validate_url`` before the browser connects, and
337 drops heavy subresources when ``block_resources`` is set. Also
338 re-scopes cookies per redirect hop (via ``_reapply_hop_cookies_sync``)
339 so a cookie set on one hop is never forwarded to a different
340 registrable domain a later hop redirects to — the same protection the
341 crawl4ai path applies via ``_reapply_hop_cookies``.
342 """
343 if not _browser_transport_allowed(self.session):
344 route.abort("blockedbyclient")
345 return
346 request = route.request
347 first_url = request.url
348 scheme = urlparse(first_url).scheme.lower()
349 if scheme not in ("http", "https"):
350 # ALLOWLIST, fail-closed: only data:/blob:/about: (never touch the
351 # network) may fall through to the browser. Every other non-http(s)
352 # scheme — file: (local file read), chrome:, gopher:, ftp: … — is
353 # aborted so a redirect or page resource cannot pivot the browser
354 # to a local file or an internal handler.
355 if scheme in _FALLTHROUGH_NON_HTTP_SCHEMES:
356 route.fallback()
357 else:
358 route.abort()
359 return
360 if (
361 self.block_resources
362 and request.resource_type in _BLOCKED_RESOURCE_TYPES
363 ):
364 route.abort()
365 return
367 current = first_url
368 method = request.method
369 # None => reuse the intercepted request's body; "" => drop the body
370 # after a POST->GET redirect downgrade (parity with safe_post).
371 body_override = None
372 # Per-hop "Cookie" header, re-derived from the browser context's
373 # cookie jar (scoped to the NEXT hop's URL) on EVERY hop by
374 # _reapply_hop_cookies_sync. It stays None only for the very first
375 # fetch, which legitimately uses the originally intercepted request's
376 # own Cookie header; from the first redirect onward it is always set
377 # (possibly to an explicit EMPTY header). This is required because
378 # route.fetch() otherwise keeps resending the ORIGINAL intercepted
379 # request's Cookie header for the rest of this handler invocation —
380 # so without a per-hop override a cookie scoped to hop N's domain
381 # would ride along to a cross-host hop N+1 (a cross-origin cookie
382 # leak). Mirrors the crawl4ai path's _reapply_hop_cookies.
383 cookie_header_override = None
384 for _ in range(_MAX_REDIRECTS + 1):
385 if not ssrf_validator.validate_url(
386 current,
387 allow_private_ips=self.allow_private_ips,
388 block_link_local=self.block_link_local,
389 ):
390 # Log scheme://host:port only — a request URL may carry
391 # credentials in userinfo / query params (RFC 3986 §3.2.1).
392 logger.warning(
393 "Playwright: blocked SSRF-unsafe browser request to {}",
394 ssrf_validator.redact_url_for_log(current),
395 )
396 route.abort("blockedbyclient")
397 return
398 fetch_kwargs = {
399 "url": current,
400 "method": method,
401 "max_redirects": 0,
402 }
403 if body_override is not None:
404 fetch_kwargs["post_data"] = body_override
405 try:
406 self._apply_hop_cookie_header(
407 request, fetch_kwargs, cookie_header_override
408 )
409 except Exception:
410 # Fail CLOSED: building the hop headers reads
411 # request.headers and must never let an exception escape
412 # this route handler and fall through to an unguarded
413 # fetch — abort the hop exactly like a failed fetch below.
414 logger.opt(exception=False).debug(
415 "Playwright: failed to build guarded hop headers"
416 )
417 route.abort("failed")
418 return
419 try:
420 response = route.fetch(**fetch_kwargs)
421 except Exception:
422 logger.opt(exception=False).debug(
423 "Playwright: guarded fetch failed"
424 )
425 route.abort("failed")
426 return
427 next_url = self._redirect_target(current, response)
428 # Re-apply this hop's Set-Cookie header(s) to the context AND
429 # re-derive, from the context jar scoped to the NEXT hop, the
430 # explicit Cookie header for that hop's route.fetch() — BEFORE
431 # following it. next_url is None on the terminal hop (no next
432 # request needs a header). See _reapply_hop_cookies for why the
433 # per-hop re-derivation is required.
434 cookie_header_override = self._reapply_hop_cookies_sync(
435 route,
436 current,
437 response,
438 next_url,
439 )
440 if next_url is None:
441 route.fulfill(response=response)
442 return
443 new_method = _resolve_redirect_method(method, response.status)
444 if new_method == "GET" and method != "GET":
445 body_override = "" # POST->GET downgrade must not leak body
446 method = new_method
447 current = next_url
448 logger.warning(
449 "Playwright: too many redirects — blocking browser request"
450 )
451 route.abort("blockedbyclient")
453 def _playwright_ws_guard(self, ws_route) -> None:
454 """Block every WebSocket on the plain-Playwright path.
456 ``route("**/*")`` does not match WS upgrades, so page JS could open
457 ``ws(s)://<internal>`` unvalidated. WebSockets are not needed for
458 HTML text extraction, so reject all of them: simply returning
459 WITHOUT calling ``connect_to_server`` means the connection to the
460 real server is never established (verified: the target sees zero
461 connections). We deliberately do NOT call ``ws_route.close()`` here —
462 in Playwright's SYNC API a blocking route-handler call re-enters the
463 dispatcher thread that ``page.goto`` is parked on and DEADLOCKS; the
464 no-op form blocks the server just as effectively without that risk.
465 """
466 try:
467 logger.debug(
468 "Playwright: blocking WebSocket to {}",
469 ssrf_validator.redact_url_for_log(ws_route.url),
470 )
471 except Exception:
472 logger.debug("Playwright: blocking WebSocket")
474 async def _crawl4ai_route_guard(self, route) -> None:
475 """Egress guard for the Crawl4AI (async Playwright) browser path.
477 Same policy as ``_playwright_route_guard`` but async: the blocking
478 DNS resolution inside ``validate_url`` is offloaded to a thread so it
479 does not stall Crawl4AI's event loop. Installed on each page context
480 via the ``on_page_context_created`` hook.
481 """
482 if not _browser_transport_allowed(self.session):
483 await route.abort("blockedbyclient")
484 return
485 request = route.request
486 first_url = request.url
487 scheme = urlparse(first_url).scheme.lower()
488 if scheme not in ("http", "https"):
489 # ALLOWLIST, fail-closed (see _playwright_route_guard): only
490 # data:/blob:/about: fall through; every other non-http(s) scheme
491 # (file:/chrome:/gopher: …) is aborted so it cannot pivot the
492 # browser to a local file or an internal handler.
493 if scheme in _FALLTHROUGH_NON_HTTP_SCHEMES:
494 await route.fallback()
495 else:
496 await route.abort()
497 return
498 if (
499 self.block_resources
500 and request.resource_type in _BLOCKED_RESOURCE_TYPES
501 ):
502 await route.abort()
503 return
505 loop = asyncio.get_running_loop()
506 current = first_url
507 method = request.method
508 body_override = None
509 # Carries the "Cookie" header for the NEXT hop, re-derived from the
510 # context's cookie jar (scoped to that hop's URL) on EVERY hop by
511 # _reapply_hop_cookies. It stays None only for the very first fetch,
512 # which legitimately uses the originally intercepted request's own
513 # Cookie header; from the first redirect onward it is always set
514 # (possibly to an explicit EMPTY header). This is required because
515 # route.fetch() otherwise keeps resending the ORIGINAL intercepted
516 # request's Cookie header for the rest of this handler invocation —
517 # so without a per-hop override a cookie scoped to hop N's domain
518 # would ride along to hop N+1 even across a DIFFERENT registrable
519 # domain (a cross-origin cookie leak).
520 cookie_header_override = None
521 for _ in range(_MAX_REDIRECTS + 1): 521 ↛ 589line 521 didn't jump to line 589 because the loop on line 521 didn't complete
522 ok = await loop.run_in_executor(
523 None,
524 functools.partial(
525 ssrf_validator.validate_url,
526 current,
527 allow_private_ips=self.allow_private_ips,
528 block_link_local=self.block_link_local,
529 ),
530 )
531 if not ok:
532 logger.warning(
533 "Crawl4AI: blocked SSRF-unsafe browser request to {}",
534 ssrf_validator.redact_url_for_log(current),
535 )
536 await route.abort("blockedbyclient")
537 return
538 fetch_kwargs = {
539 "url": current,
540 "method": method,
541 "max_redirects": 0,
542 }
543 if body_override is not None:
544 fetch_kwargs["post_data"] = body_override
545 try:
546 self._apply_hop_cookie_header(
547 request, fetch_kwargs, cookie_header_override
548 )
549 except Exception:
550 # Fail CLOSED: building the hop headers reads
551 # request.headers and must never let an exception escape
552 # this route handler and fall through to an unguarded
553 # fetch — abort the hop exactly like a failed fetch below.
554 logger.opt(exception=False).debug(
555 "Crawl4AI: failed to build guarded hop headers"
556 )
557 await route.abort("failed")
558 return
559 try:
560 response = await route.fetch(**fetch_kwargs)
561 except Exception:
562 logger.opt(exception=False).debug(
563 "Crawl4AI: guarded fetch failed"
564 )
565 await route.abort("failed")
566 return
567 next_url = self._redirect_target(current, response)
568 # Re-apply this hop's Set-Cookie header(s) to the context AND
569 # re-derive, from the context jar scoped to the NEXT hop, the
570 # explicit Cookie header for that hop's route.fetch() — BEFORE
571 # following it. Passing next_url (which is None on the terminal
572 # hop) lets _reapply_hop_cookies skip deriving a header when
573 # there is no next request. See _reapply_hop_cookies for why the
574 # per-hop re-derivation is required on the crawl4ai path.
575 cookie_header_override = await self._reapply_hop_cookies(
576 route,
577 current,
578 response,
579 next_url,
580 )
581 if next_url is None:
582 await route.fulfill(response=response)
583 return
584 new_method = _resolve_redirect_method(method, response.status)
585 if new_method == "GET" and method != "GET":
586 body_override = "" # POST->GET downgrade must not leak body
587 method = new_method
588 current = next_url
589 logger.warning(
590 "Crawl4AI: too many redirects — blocking browser request"
591 )
592 await route.abort("blockedbyclient")
594 async def _crawl4ai_ws_guard(self, ws_route) -> None:
595 """Block every WebSocket on the Crawl4AI path (async twin of
596 ``_playwright_ws_guard``).
598 Same no-op-reject strategy: not calling ``connect_to_server`` leaves
599 the real server unreached. Kept no-op (no ``close()``) for symmetry
600 with the sync guard and to avoid any handler-reentrancy surprises.
601 """
602 try:
603 logger.debug(
604 "Crawl4AI: blocking WebSocket to {}",
605 ssrf_validator.redact_url_for_log(ws_route.url),
606 )
607 except Exception:
608 logger.debug("Crawl4AI: blocking WebSocket")
610 @staticmethod
611 def _cookies_from_response(response, response_url: str) -> list:
612 """Parse ``Set-Cookie`` response header(s) into Playwright cookie
613 dicts, ready for ``BrowserContext.add_cookies()``.
615 WHY THIS EXISTS: both browser guards re-derive the ``Cookie`` request
616 header per redirect hop from the context's own jar, so this parses a
617 hop's ``Set-Cookie`` header(s) and re-applies them to the jar before
618 the next hop is followed. On the crawl4ai path it is also load-bearing
619 for content fidelity: crawl4ai pre-seeds the page context with a
620 marker cookie via ``context.add_cookies()`` before navigation
621 (``BrowserManager.setup_context``), and once a context's jar has been
622 touched that way, Playwright's ``route.fetch()`` on a subsequent
623 redirect hop does NOT merge that hop's ``Set-Cookie`` into the jar for
624 us — so a session/auth cookie set on an intermediate hop would
625 silently be lost, corrupting the final page (e.g. a login- or
626 region-gated redirect).
628 A ``Set-Cookie`` that is really a DELETION directive (``Max-Age=0`` or
629 a past ``Expires``) is honored and NOT re-added, so a cookie the
630 server just cleared is never resurrected onto the jar.
632 Uses ``headers_array`` (a list of ``{name, value}`` pairs) rather
633 than the ``headers`` dict, because a response can carry MULTIPLE
634 ``Set-Cookie`` headers — a plain dict would silently collapse them
635 to one.
636 """
637 host = urlparse(response_url).hostname
638 if not host: 638 ↛ 639line 638 didn't jump to line 639 because the condition on line 638 was never true
639 return []
640 try:
641 headers = response.headers_array
642 except Exception:
643 return []
644 cookies = []
645 for header in headers:
646 if str(header.get("name", "")).lower() != "set-cookie":
647 continue
648 raw = header.get("value", "")
649 try:
650 parsed = _parse_one_set_cookie(raw)
651 except Exception:
652 logger.opt(exception=False).debug(
653 "Browser guard: failed to parse a Set-Cookie header"
654 )
655 continue
656 if parsed is None:
657 continue
658 name, value, attrs = parsed
659 if _set_cookie_is_expired(attrs):
660 # A Max-Age=0 / past-Expires Set-Cookie is a DELETION
661 # directive: re-adding it would resurrect a cookie the
662 # server just cleared. Honor the expiry and skip it.
663 logger.debug(
664 "Browser guard: skipping expired/deletion Set-Cookie"
665 )
666 continue
667 raw_domain = attrs.get("domain")
668 if (
669 isinstance(raw_domain, str)
670 and raw_domain
671 and _cookie_domain_matches(host, raw_domain)
672 ):
673 cookie_domain = raw_domain
674 else:
675 # No Domain attribute, or a Domain that does NOT
676 # domain-match the response host (a cross-domain or
677 # unrelated Domain a browser rejects). Scope the cookie
678 # to the response host — mirroring browser behavior and
679 # preventing a hop from planting a cookie on another
680 # registrable domain (session fixation).
681 if isinstance(raw_domain, str) and raw_domain:
682 logger.debug(
683 "Browser guard: rejecting a Set-Cookie Domain that "
684 "does not match the response host; scoping the "
685 "cookie to the host instead"
686 )
687 cookie_domain = host
688 path = attrs.get("path")
689 cookie = {
690 "name": name,
691 "value": value,
692 "domain": cookie_domain,
693 "path": path if isinstance(path, str) and path else "/",
694 }
695 if attrs.get("secure"):
696 cookie["secure"] = True
697 if attrs.get("httponly"):
698 cookie["httpOnly"] = True
699 cookies.append(cookie)
700 return cookies
702 async def _reapply_hop_cookies(
703 self,
704 route,
705 current_url: str,
706 response,
707 next_hop_url: Optional[str],
708 ) -> Optional[str]:
709 """Apply ``response``'s ``Set-Cookie`` header(s) to the browser
710 context and return the ``Cookie`` header the caller must pass to the
711 NEXT hop's ``route.fetch(headers=...)`` — re-derived, on EVERY hop,
712 from the context jar scoped to ``next_hop_url``.
714 Why re-derive on every hop instead of carrying the previous hop's
715 header forward: ``route.fetch()`` keeps resending the ORIGINAL
716 intercepted request's Cookie header for the rest of this guard
717 invocation unless we override it. If we ever carried a previous
718 hop's Cookie header (or let the original one ride along) across a
719 redirect to a DIFFERENT registrable domain, that domain would
720 receive cookies scoped to the previous one — a cross-origin cookie
721 leak a real browser never performs. So we ALWAYS read the jar back
722 through ``context.cookies(next_hop_url)`` (Playwright scopes it by
723 domain/path) and return exactly those cookies:
725 * same-domain next hop -> the jar returns that domain's cookies
726 (including any just set on this hop, applied below via
727 ``add_cookies``), so same-domain persistence keeps working;
728 * cross-domain next hop -> the jar returns nothing for that domain,
729 so we return an EMPTY header and the caller sends NO Cookie to it.
731 ``context.add_cookies()`` still runs for its own sake: it keeps the
732 jar correct for anything reading it independently of this walk
733 (subresources of the eventually-fulfilled page, ``document.cookie``,
734 later navigations in the same context).
736 Fail-SAFE: on any parse/apply/read error we return an EMPTY header
737 rather than a previous hop's, so an error can only DROP a cookie,
738 never leak one cross-domain. This never affects the SSRF
739 validate/abort decisions around it. Returns ``None`` only for the
740 terminal hop (``next_hop_url is None`` — no next request needs a
741 header); that value is unused by the caller.
742 """
743 try:
744 context = route.request.frame.page.context
745 cookies = self._cookies_from_response(response, current_url)
746 if cookies:
747 await context.add_cookies(cookies)
748 if next_hop_url is None:
749 # Terminal hop: no next request, so no Cookie header to
750 # derive. The jar was still updated above.
751 return None
752 live_cookies = await context.cookies(next_hop_url)
753 return "; ".join(f"{c['name']}={c['value']}" for c in live_cookies)
754 except Exception:
755 logger.opt(exception=False).debug(
756 "Crawl4AI: failed to re-apply redirect-hop cookies"
757 )
758 # Prefer sending NO cookie to the next hop over resending a
759 # previous hop's (possibly cross-domain) one: "" clears the
760 # Cookie header. None is used only when there is no next hop.
761 return "" if next_hop_url is not None else None
763 def _reapply_hop_cookies_sync(
764 self,
765 route,
766 current_url: str,
767 response,
768 next_hop_url: Optional[str],
769 ) -> Optional[str]:
770 """Synchronous twin of ``_reapply_hop_cookies`` for the plain
771 (sync-API) Playwright path.
773 Same contract and fail-SAFE behavior — see ``_reapply_hop_cookies``
774 for the full rationale. Applies this hop's ``Set-Cookie`` header(s)
775 to the browser context and returns the ``Cookie`` header for the NEXT
776 hop, re-derived from the context jar scoped to ``next_hop_url`` so a
777 cookie belonging to one domain is never forwarded to a different
778 registrable domain a later redirect lands on. Returns ``None`` only
779 for the terminal hop; on any error returns an EMPTY header (drop a
780 cookie, never leak one cross-domain).
781 """
782 try:
783 context = route.request.frame.page.context
784 cookies = self._cookies_from_response(response, current_url)
785 if cookies:
786 context.add_cookies(cookies)
787 if next_hop_url is None:
788 # Terminal hop: no next request, so no Cookie header to
789 # derive. The jar was still updated above.
790 return None
791 live_cookies = context.cookies(next_hop_url)
792 return "; ".join(f"{c['name']}={c['value']}" for c in live_cookies)
793 except Exception:
794 logger.opt(exception=False).debug(
795 "Playwright: failed to re-apply redirect-hop cookies"
796 )
797 return "" if next_hop_url is not None else None
799 @staticmethod
800 def _apply_hop_cookie_header(
801 request, fetch_kwargs: dict, cookie_header_override: Optional[str]
802 ) -> None:
803 """Override ONLY the ``Cookie`` header on ``fetch_kwargs`` with the
804 per-hop re-derived ``cookie_header_override``.
806 Shared by both browser guards (sync + crawl4ai). Rebuilds the base
807 headers ``route.fetch()`` would otherwise default to (the original
808 intercepted request's) MINUS its ``Cookie`` header, then sets the
809 freshly re-derived Cookie header. When that value is EMPTY, the Cookie
810 header is dropped entirely rather than sent empty: the next hop's
811 domain has no cookies in the jar, so sending nothing is correct — and,
812 crucially, this stops ``route.fetch()`` from resending the ORIGINAL
813 intercepted request's Cookie header to what may be a DIFFERENT
814 registrable domain (the cross-origin leak this guards against).
816 The original request's ``Host`` and ``Content-Length`` are ALSO dropped
817 from the rebuilt headers on every hop: a redirect hop's ``Host`` must be
818 derived from the NEW target (a forwarded stale ``Host`` is wrong on a
819 cross-host hop), and after a POST->GET downgrade (body dropped to "")
820 the original ``Content-Length`` no longer matches the body.
821 ``route.fetch()`` recomputes both from the hop URL and the actual body
822 once these headers are absent.
824 On a CROSS-SITE hop the originally intercepted request's
825 ``Authorization`` / ``Proxy-Authorization`` headers are ALSO dropped
826 (in addition to ``Cookie``): those credentials are scoped to the
827 original origin, so — exactly as a real browser strips Authorization
828 on a cross-origin redirect — they must never ride along to a different
829 registrable domain a later hop lands on. Same-site hops keep them so
830 legitimate within-site auth redirects (e.g. api.example.com ->
831 example.com) still work. Cross-site is decided by
832 ``_same_registrable_site`` — the same host/subdomain notion the cookie
833 re-scoping relies on.
835 A ``None`` override (the very first hop only) leaves ``fetch_kwargs``
836 untouched so the initial request keeps its own Cookie header. The very
837 first hop is by construction same-origin (its target IS the original
838 request URL), so no credential is ever leaked by that early return.
839 """
840 if cookie_header_override is None:
841 return
842 # Dropped on EVERY (post-first) hop, before rebuilding the headers:
843 # * cookie — always re-derived per hop below.
844 # * host — a redirect hop's Host must come from the NEW target, not a
845 # stale forwarded value; a cross-host hop that kept the previous
846 # hop's Host would send the wrong Host. ``route.fetch()`` derives the
847 # correct Host from the hop URL once this header is absent.
848 # * content-length — after a POST->GET redirect downgrade the body is
849 # dropped to "" (see the guard loop's ``body_override``), so a
850 # forwarded original Content-Length would be incorrect;
851 # ``route.fetch()`` recomputes it from the actual (post_data) body.
852 drop = {"cookie", "host", "content-length"}
853 # On a cross-site hop also strip the credential headers the original
854 # request carried (a browser strips Authorization cross-origin too).
855 origin_host = urlparse(request.url).hostname
856 target_host = urlparse(fetch_kwargs.get("url", "")).hostname
857 if not _same_registrable_site(origin_host, target_host):
858 drop |= {"authorization", "proxy-authorization"}
859 base_headers = {
860 k: v for k, v in request.headers.items() if k.lower() not in drop
861 }
862 if cookie_header_override:
863 base_headers["cookie"] = cookie_header_override
864 fetch_kwargs["headers"] = base_headers
866 @staticmethod
867 def _redirect_target(current_url: str, response) -> Optional[str]:
868 """Return the absolute redirect target for ``response``, or None.
870 None means the response is terminal (not a followed 3xx, or a 3xx
871 with no ``Location``) and should be served to the browser as-is.
872 Mirrors the static path's redirect handling so both validate the
873 same hops.
874 """
875 if response.status not in _REDIRECT_STATUS_CODES:
876 return None
877 location = response.headers.get("location")
878 if not location: 878 ↛ 879line 878 didn't jump to line 879 because the condition on line 878 was never true
879 return None
880 return urljoin(current_url, location.strip())
882 def _robots_allows(self, url: str) -> bool:
883 """Guarded robots.txt politeness check.
885 REPLACES Crawl4AI's built-in ``check_robots_txt`` fetch, which used a
886 raw ``aiohttp`` client (redirect-following, no SSRF check) — a
887 ``/robots.txt`` that 302-redirected to an internal/metadata host was
888 fetched unguarded. Here the fetch goes through the downloader's
889 ``SafeSession``, whose ``send()`` SSRF-validates every hop (initial +
890 redirects) under the same egress scope as the browser guard.
892 Fails OPEN (returns True) on any non-200, network error, SSRF block,
893 or parse failure — matching the "allow on robots uncertainty"
894 convention. It only ever DISALLOWS on an explicit robots rule, so a
895 blocked (SSRF-unsafe) robots fetch never grants extra access.
896 """
897 from urllib.robotparser import RobotFileParser
899 try:
900 parsed = urlparse(url)
901 if not parsed.netloc: 901 ↛ 902line 901 didn't jump to line 902 because the condition on line 901 was never true
902 return True
903 scheme = parsed.scheme or "http"
904 robots_url = f"{scheme}://{parsed.netloc}/robots.txt"
905 response = self.session.get(
906 robots_url,
907 timeout=min(self.timeout, 10),
908 allow_redirects=True,
909 )
910 if response.status_code != 200 or not response.text: 910 ↛ 911line 910 didn't jump to line 911 because the condition on line 910 was never true
911 return True
912 parser = RobotFileParser()
913 parser.parse(response.text.splitlines())
914 return parser.can_fetch(BROWSER_USER_AGENT, url)
915 except PolicyDeniedError:
916 raise
917 except Exception:
918 # SafeSession raises ValueError on an SSRF-unsafe hop; network /
919 # parse errors land here too. Fail open for politeness — the
920 # actual navigation is still validated by the browser guard.
921 logger.opt(exception=False).debug(
922 "Guarded robots.txt check failed open"
923 )
924 return True
926 def _fetch_html(self, url: str) -> Optional[str]:
927 """Fetch HTML with JS rendering.
929 Tries Crawl4AI first (with robots.txt, shadow DOM, iframes),
930 falls back to plain Playwright.
931 """
932 if not _browser_transport_allowed(self.session):
933 raise PolicyDeniedError(
934 Decision(False, "browser_transport_cannot_enforce_scope"), url
935 )
936 # Entry-URL scheme guard (fail-closed). The browser downloader is
937 # reachable via download()/download_with_result(), which do NOT
938 # constrain the scheme themselves — only can_handle() checks it, and
939 # not every caller consults can_handle() (e.g.
940 # extraction/pipeline.py calls download() directly). The per-request
941 # route guard validates redirect hops and subresources, but a
942 # top-level page.goto()/crawler.arun() on a non-http(s) entry URL
943 # (e.g. file:///etc/passwd) could read a LOCAL FILE and return it as
944 # page content before any hop is walked. Refuse non-http(s) entry URLs
945 # here so such a URL is never navigated.
946 if urlparse(url).scheme.lower() not in ("http", "https"):
947 logger.warning(
948 "Browser downloader: refusing non-http(s) entry URL {}",
949 ssrf_validator.redact_url_for_log(url),
950 )
951 return None
952 # Try Crawl4AI first (richer features, robots.txt)
953 html = self._fetch_with_crawl4ai(url)
954 if html is not None: 954 ↛ 961line 954 didn't jump to line 961 because the condition on line 954 was always true
955 # Crawl4AI succeeded (non-empty) or intentionally blocked
956 # by robots.txt (empty string). Either way, don't fall
957 # through to Playwright.
958 return html or None
960 # Crawl4AI not installed or failed — fall back to Playwright
961 return self._fetch_with_playwright(url)
963 def _fetch_with_crawl4ai(self, url: str) -> Optional[str]:
964 """Fetch HTML using Crawl4AI with ethical defaults."""
965 engine_type = f"crawl4ai_download_{rate_limit_authority(url)}"
967 try:
968 from crawl4ai import (
969 AsyncWebCrawler,
970 BrowserConfig,
971 CrawlerRunConfig,
972 )
973 except ImportError:
974 logger.debug(
975 "crawl4ai.unavailable url={url}",
976 url=redact_url_for_log(url),
977 )
978 return None
980 # Ethical robots.txt check — done HERE through the guarded
981 # SafeSession, NOT via crawl4ai's check_robots_txt (disabled below):
982 # crawl4ai fetches robots.txt with a raw aiohttp client that follows
983 # redirects with no SSRF validation, so a redirecting /robots.txt is
984 # an SSRF vector that bypasses the browser route guard entirely.
985 if not self._robots_allows(url):
986 logger.info(
987 "Crawl4AI: blocked by robots.txt for {}",
988 ssrf_validator.redact_url_for_log(url),
989 )
990 return "" # Empty string signals intentional skip
992 logger.debug(
993 "crawl4ai.fetch_started url={url}",
994 url=redact_url_for_log(url),
995 )
996 wait_time = self.rate_tracker.apply_rate_limit(engine_type)
998 browser_cfg = BrowserConfig(
999 headless=True,
1000 verbose=False,
1001 user_agent=BROWSER_USER_AGENT,
1002 )
1003 run_cfg = CrawlerRunConfig(
1004 # robots.txt handled by the guarded _robots_allows() above;
1005 # crawl4ai's own fetch is unguarded (SSRF), so keep it OFF.
1006 check_robots_txt=False,
1007 # Better extraction: flatten modern web features
1008 flatten_shadow_dom=True,
1009 process_iframes=True,
1010 # Trigger lazy-loaded content
1011 scan_full_page=True,
1012 # Performance
1013 wait_until=self.wait_until,
1014 page_timeout=self.timeout * 1000,
1015 exclude_all_images=self.block_resources,
1016 # No stealth
1017 override_navigator=False,
1018 magic=False,
1019 simulate_user=False,
1020 verbose=False,
1021 )
1023 try:
1025 async def _crawl():
1026 async with AsyncWebCrawler(config=browser_cfg) as crawler:
1027 # Install the SSRF route guard on every page context
1028 # BEFORE any navigation. FAIL-SAFE: if the hook API is
1029 # unavailable (crawl4ai API drift), do NOT run crawl4ai
1030 # unguarded — return None so _fetch_html falls through
1031 # to the guarded plain-Playwright path.
1032 async def _on_page_context_created(
1033 page, context=None, config=None, **kwargs
1034 ):
1035 # Install ALL three guards fail-CLOSED: if any raises
1036 # (crawl4ai/Playwright API drift, or a context with
1037 # incomplete API parity), let it propagate so the
1038 # crawl is abandoned — crawl4ai's execute_hook does not
1039 # swallow it, so it reaches the outer ``except`` below
1040 # and _fetch_with_crawl4ai returns None, falling back
1041 # to the guarded plain-Playwright path. Never navigate
1042 # with a missing HTTP, WebSocket, or SW guard.
1043 target = context if context is not None else page
1044 # HTTP(S) requests: SSRF-validate every hop.
1045 await target.route("**/*", self._crawl4ai_route_guard)
1046 # WebSockets: route("**/*") does not match WS upgrades
1047 # — block them explicitly.
1048 await target.route_web_socket(
1049 "**/*", self._crawl4ai_ws_guard
1050 )
1051 # Service Workers: crawl4ai's BrowserConfig cannot set
1052 # service_workers="block", and SW-made requests bypass
1053 # route handlers — prevent SW registration via an init
1054 # script so all traffic stays on the guarded path.
1055 await target.add_init_script(
1056 _DISABLE_SERVICE_WORKERS_JS
1057 )
1058 # WebRTC: RTCPeerConnection ICE (STUN/TURN) opens raw
1059 # UDP/TCP sockets via Chromium's WebRTC stack, which
1060 # neither route() nor route_web_socket() intercept —
1061 # neutralize it before any page script runs.
1062 await target.add_init_script(_DISABLE_WEBRTC_JS)
1063 return page
1065 try:
1066 crawler.crawler_strategy.set_hook(
1067 "on_page_context_created",
1068 _on_page_context_created,
1069 )
1070 except Exception:
1071 logger.warning(
1072 "Crawl4AI: could not install SSRF route guard — "
1073 "refusing to crawl unguarded, falling back to "
1074 "the guarded Playwright path"
1075 )
1076 return None
1077 return await crawler.arun(url=url, config=run_cfg)
1079 # A guard-install failure raised inside the hook propagates out of
1080 # crawl4ai's execute_hook and arun, landing in the outer ``except``
1081 # below (return None -> guarded plain-Playwright fallback). This is
1082 # the fail-CLOSED path: a crawl NEVER proceeds with a missing HTTP,
1083 # WebSocket, or Service-Worker guard.
1084 result = _run_async(_crawl(), timeout=self.timeout + 30)
1086 if result is None:
1087 # Guard could not be installed (fail-safe) — defer to the
1088 # guarded plain-Playwright path rather than crawl unguarded.
1089 return None
1091 if result.success and result.html:
1092 html = result.html
1093 logger.debug(
1094 "crawl4ai.fetch_succeeded url={url} content_length={content_length}",
1095 url=redact_url_for_log(url),
1096 content_length=len(html),
1097 )
1098 self.rate_tracker.record_outcome(
1099 engine_type=engine_type,
1100 wait_time=wait_time,
1101 success=True,
1102 retry_count=1,
1103 search_result_count=1,
1104 )
1105 return html
1107 # Check if blocked by robots.txt
1108 error_msg = getattr(result, "error_message", "") or ""
1109 if "robots.txt" in error_msg.lower():
1110 logger.info(
1111 "crawl4ai.robots_denied url={url}",
1112 url=redact_url_for_log(url),
1113 )
1114 # Don't fall back to Playwright — respect the block
1115 self.rate_tracker.record_outcome(
1116 engine_type=engine_type,
1117 wait_time=wait_time,
1118 success=False,
1119 retry_count=1,
1120 error_type="robots_txt_blocked",
1121 )
1122 return "" # Empty string signals intentional skip
1124 status = getattr(result, "status_code", "unknown")
1125 logger.debug(
1126 "crawl4ai.fetch_failed url={url}",
1127 url=redact_url_for_log(url),
1128 )
1129 self.rate_tracker.record_outcome(
1130 engine_type=engine_type,
1131 wait_time=wait_time,
1132 success=False,
1133 retry_count=1,
1134 error_type=f"crawl4ai_status_{status}",
1135 )
1136 return None
1138 except PolicyDeniedError:
1139 raise
1140 except Exception as e:
1141 # DEBUG, not ERROR: this except is normal control flow — it
1142 # returns None, so _fetch_html falls back to
1143 # _fetch_with_playwright and the fetch usually then succeeds.
1144 # Logging this at ERROR would write a false-alarm entry into the
1145 # encrypted DB sink and the frontend progress stream for every
1146 # crawl4ai hiccup, even when the overall fetch didn't fail.
1147 logger.opt(exception=False).debug(
1148 "crawl4ai.fetch_error url={url} error_type={error_type}",
1149 url=redact_url_for_log(url),
1150 error_type=type(e).__name__,
1151 )
1152 self.rate_tracker.record_outcome(
1153 engine_type=engine_type,
1154 wait_time=wait_time,
1155 success=False,
1156 retry_count=1,
1157 error_type=type(e).__name__,
1158 )
1159 return None
1161 def _fetch_with_playwright(self, url: str) -> Optional[str]:
1162 """Fetch HTML using plain Playwright (fallback)."""
1163 logger.debug(
1164 "playwright.fetch_started url={url}",
1165 url=redact_url_for_log(url),
1166 )
1167 engine_type = f"playwright_download_{rate_limit_authority(url)}"
1169 wait_time = self.rate_tracker.apply_rate_limit(engine_type)
1171 try:
1172 from playwright.sync_api import sync_playwright
1174 # Lazy-init browser (reuse across multiple fetches).
1175 # --no-sandbox: Chromium needs SYS_ADMIN to set up its user-namespace
1176 # sandbox; the production container drops that cap. Without this
1177 # flag, launch() crashes inside Docker. Crawl4AI's own arg list
1178 # already includes it; this fallback path was missing it.
1179 # --disable-dev-shm-usage: Docker's default /dev/shm is 64 MB,
1180 # which Chromium can blow through and OOM. Use /tmp instead.
1181 if self._browser is None: 1181 ↛ 1204line 1181 didn't jump to line 1204 because the condition on line 1181 was always true
1182 logger.debug(
1183 "playwright.launch_started url={url}",
1184 url=redact_url_for_log(url),
1185 )
1186 pw = sync_playwright().start()
1187 try:
1188 self._browser = pw.chromium.launch(
1189 headless=True,
1190 args=["--no-sandbox", "--disable-dev-shm-usage"],
1191 )
1192 except Exception:
1193 pw.stop()
1194 raise
1195 self._playwright = pw
1197 # ``browser.new_page()`` creates a FRESH BrowserContext for this
1198 # fetch (Playwright makes the page own that context) — only
1199 # ``self._browser`` is cached across fetches, NOT the context. The
1200 # route/WS handlers and init scripts below are therefore registered
1201 # on a per-fetch context that ``page.close()`` (in the finally)
1202 # tears down together with them, so they do NOT accumulate across
1203 # fetches; each fetch gets exactly one of each guard.
1204 page = self._browser.new_page(
1205 user_agent=BROWSER_USER_AGENT,
1206 # Block Service Workers: their requests are not intercepted by
1207 # route handlers, so a SW could fetch an internal/metadata
1208 # host unvalidated. Blocking forces all traffic through the
1209 # guarded normal path (SWs aren't needed for text extraction).
1210 service_workers="block",
1211 )
1212 try:
1213 # Egress guard for EVERY request: validates the initial
1214 # navigation and each redirect hop before the browser
1215 # connects (see _playwright_route_guard), and also drops
1216 # heavy subresources when block_resources is set — so it
1217 # subsumes the old extension-glob resource route. Registered
1218 # at the CONTEXT level so it also covers requests a blocked
1219 # SW would otherwise have made.
1220 page.context.route("**/*", self._playwright_route_guard)
1221 # WebSockets bypass route("**/*") — block them explicitly.
1222 page.context.route_web_socket("**/*", self._playwright_ws_guard)
1223 # WebRTC (RTCPeerConnection ICE/STUN/TURN) opens raw sockets via
1224 # Chromium's WebRTC stack, which no route handler intercepts —
1225 # neutralize it before any page script runs. Install at the
1226 # CONTEXT level (like the route guards above) so it also covers
1227 # popups (window.open) — a page-level init script does not
1228 # propagate to new pages, leaving a popup with live WebRTC.
1229 page.context.add_init_script(_DISABLE_WEBRTC_JS)
1231 page.goto(
1232 url,
1233 wait_until=self.wait_until,
1234 timeout=self.timeout * 1000,
1235 )
1236 html = page.content()
1237 finally:
1238 try:
1239 page.close()
1240 except Exception:
1241 logger.debug(
1242 "playwright.page_close_failed url={url}",
1243 url=redact_url_for_log(url),
1244 )
1246 if html:
1247 logger.debug(
1248 "playwright.fetch_succeeded url={url} content_length={content_length}",
1249 url=redact_url_for_log(url),
1250 content_length=len(html),
1251 )
1252 self.rate_tracker.record_outcome(
1253 engine_type=engine_type,
1254 wait_time=wait_time,
1255 success=True,
1256 retry_count=1,
1257 search_result_count=1,
1258 )
1259 return html
1261 logger.debug(
1262 "playwright.fetch_empty url={url}",
1263 url=redact_url_for_log(url),
1264 )
1265 return None
1267 except ImportError:
1268 logger.warning(
1269 "playwright.unavailable url={url}",
1270 url=redact_url_for_log(url),
1271 )
1272 return None
1273 except PolicyDeniedError:
1274 raise
1275 except Exception as e:
1276 # Type name only: Playwright error texts embed the navigated
1277 # URL, and loguru renders ``ExcType: str(e)`` with a traceback
1278 # even when diagnose=False.
1279 logger.opt(exception=False).error(
1280 "playwright.fetch_error url={url} error_type={error_type}",
1281 url=redact_url_for_log(url),
1282 error_type=type(e).__name__,
1283 )
1284 self.rate_tracker.record_outcome(
1285 engine_type=engine_type,
1286 wait_time=wait_time,
1287 success=False,
1288 retry_count=1,
1289 error_type=type(e).__name__,
1290 )
1291 return None
1293 def close(self):
1294 """Clean up Playwright browser and resources."""
1295 if self._browser:
1296 try:
1297 self._browser.close()
1298 except Exception:
1299 logger.debug("playwright.browser_close_failed")
1300 self._browser = None
1301 if self._playwright:
1302 try:
1303 self._playwright.stop()
1304 except Exception:
1305 logger.debug("playwright.stop_failed")
1306 self._playwright = None
1307 super().close()
1310class AutoHTMLDownloader(HTMLDownloader):
1311 """HTML downloader that tries static fetch first, falls back to
1312 Crawl4AI/Playwright when the page needs JavaScript rendering.
1314 Detection heuristics:
1315 - Extracted content is too short (<200 chars)
1316 - Raw HTML contains SPA framework signals (React, Vue, Angular, Next.js)
1317 """
1319 def __init__(
1320 self,
1321 timeout: int = 30,
1322 language: str = "English",
1323 min_content_length: int = 200,
1324 # Disabled by default to match the production Docker image, which
1325 # ships without Chromium — every JS-rendering fallback attempt
1326 # would otherwise fail loudly (see issue #3826). Callers running
1327 # outside Docker with Chromium installed opt in via the
1328 # ``web.enable_javascript_rendering`` setting, or pass ``True``
1329 # explicitly when constructing the downloader.
1330 enable_js_rendering: bool = False,
1331 allow_private_ips: bool = False,
1332 block_link_local: bool = False,
1333 **kwargs,
1334 ):
1335 super().__init__(timeout=timeout, language=language)
1336 self.min_content_length = min_content_length
1337 self.enable_js_rendering = enable_js_rendering
1338 # SSRF policy threaded to the lazily-built Playwright child so the
1339 # JS-render browser path honours the active egress scope. Default
1340 # strict; ContentFetcher relaxes to True under PRIVATE_ONLY.
1341 self.allow_private_ips = allow_private_ips
1342 # Keep the static-fetch SafeSession in lockstep with the browser
1343 # path so both routes honour the same private-egress gate (mirrors
1344 # PlaywrightHTMLDownloader.__init__).
1345 if hasattr(self.session, "allow_private_ips"): 1345 ↛ 1353line 1345 didn't jump to line 1353 because the condition on line 1345 was always true
1346 self.session.allow_private_ips = allow_private_ips
1347 # Link-local carve-out inside the private-IP grant: cloud metadata
1348 # lives in the link-local range beyond the always-blocked literals
1349 # and no legitimate page does. Off by default; full_search sets it
1350 # alongside allow_private_ips. The SafeSession forwards it to every
1351 # hop of the static fetch and the Playwright child to both of its
1352 # route guards.
1353 self.block_link_local = block_link_local
1354 if hasattr(self.session, "block_link_local"): 1354 ↛ 1356line 1354 didn't jump to line 1356 because the condition on line 1354 was always true
1355 self.session.block_link_local = block_link_local
1356 self._playwright_downloader = None
1357 # Host-level attempt budget for the fallback chain. The static leg
1358 # backs off adaptively and bounds its retries, but its 429/503
1359 # outcomes never reach the later legs: the probe GET applies no
1360 # rate limiting, and the browser leg backs off only on its own
1361 # tracker buckets (playwright_download_/crawl4ai_download_), which
1362 # never see the static leg's html_download_ outcomes. When the
1363 # static leg gives up with HTTP 429/503, the host's authority is
1364 # marked here for the rest of this downloader's life
1365 # (fetch_and_extract creates one per URL; batch_fetch_and_extract
1366 # and ContentFetcher share one across several URLs), and the probe
1367 # and browser legs are skipped for that host from then on. The
1368 # static leg still runs for each later URL, and the mark never
1369 # rewrites that URL's result. One user action must not keep
1370 # sending requests to a host that is asking us to stop.
1371 self._rate_limited_hosts: set[str] = set()
1372 # Whether the static fetch of the URL that download_with_result()
1373 # is working on gave up on 429/503 itself, as opposed to an earlier
1374 # URL having marked its host. It is per call (download_with_result()
1375 # clears it on entry), and is what lets that method report a
1376 # rate-limited skip reason only for the URL that earned it.
1377 self._static_fetch_rate_limited = False
1379 def _host_rate_limited(self, url: str) -> bool:
1380 """Whether a static fetch already gave up on 429/503 for this host."""
1381 return rate_limit_authority(url) in self._rate_limited_hosts
1383 def _note_host_rate_limited(self, url: str) -> None:
1384 """Record that the static leg exhausted its retries on 429/503.
1386 Marks the host for the rest of this downloader's life and flags the
1387 fetch in progress (``_static_fetch_rate_limited``).
1388 """
1389 self._rate_limited_hosts.add(rate_limit_authority(url))
1390 self._static_fetch_rate_limited = True
1392 def _get_playwright_downloader(self) -> PlaywrightHTMLDownloader:
1393 """Lazy-init JS rendering downloader for fallback."""
1394 if self._playwright_downloader is None:
1395 self._playwright_downloader = PlaywrightHTMLDownloader(
1396 timeout=self.timeout,
1397 language=self.language,
1398 allow_private_ips=self.allow_private_ips,
1399 block_link_local=self.block_link_local,
1400 )
1401 # The child creates its own SafeSession for robots.txt and other
1402 # auxiliary requests. Keep it bound to the parent run's policy.
1403 self._playwright_downloader.session.egress_context = getattr(
1404 self.session, "egress_context", None
1405 )
1406 return self._playwright_downloader
1408 @staticmethod
1409 def _has_spa_signals(html: str) -> bool:
1410 """Check if HTML contains signals of a JS-rendered SPA."""
1411 html_lower = html[:5000].lower() # Only check head/early body
1412 return any(signal.lower() in html_lower for signal in SPA_SIGNALS)
1414 def _fetch_html(self, url: str) -> Optional[str]:
1415 """Fetch HTML statically, storing raw response for SPA detection.
1417 Note: _last_raw_html is instance state read by download()/download_with_result().
1418 This is safe because AutoHTMLDownloader instances are created per-request
1419 in fetch_and_extract/batch_fetch_and_extract — not shared across threads.
1420 """
1421 self._last_raw_html = None
1422 # Try the normal static fetch. ``on_rate_limited`` fires when the
1423 # static leg exhausts its retries on 429/503, so the host is
1424 # marked before any later leg runs.
1425 html = super()._fetch_html(
1426 url, on_rate_limited=lambda: self._note_host_rate_limited(url)
1427 )
1428 if html:
1429 self._last_raw_html = html
1430 return html
1432 # Static fetch hit recoverable non-HTML content (PDF/text stash for
1433 # download() recovery): a second raw GET would just re-download the
1434 # same bytes, so skip the SPA probe entirely.
1435 if ( 1435 ↛ 1439line 1435 didn't jump to line 1439 because the condition on line 1435 was never true
1436 getattr(self, "_recovery_payload", None) is not None
1437 and getattr(self, "_recovery_request_url", None) == url
1438 ):
1439 return None
1441 # Host-level budget: a static fetch to this host gave up on
1442 # 429/503 — do not spend the probe request on a host that asked us
1443 # to stop.
1444 if self._host_rate_limited(url):
1445 logger.debug(
1446 "auto_html.host_rate_limited_probe_skipped url={url}",
1447 url=redact_url_for_log(url),
1448 )
1449 return None
1451 # Static fetch failed (403, etc.) — try raw GET to check for
1452 # challenge pages / SPA signals even on non-200 responses
1453 try:
1454 response = self.session.get(
1455 url,
1456 timeout=self.timeout,
1457 allow_redirects=True,
1458 )
1459 self._last_raw_html = response.text
1460 except PolicyDeniedError:
1461 raise
1462 except Exception:
1463 # Expected/routine (403s, challenge pages, network blips) —
1464 # keep at DEBUG, not ERROR, to avoid flooding alerting. Still
1465 # drop the exception attachment: a network error here can embed
1466 # the raw/credentialed URL or a cookie value.
1467 logger.opt(exception=False).debug(
1468 "auto_html.raw_fetch_failed url={url}",
1469 url=redact_url_for_log(url),
1470 )
1471 return None
1473 def download(self, url, content_type=None):
1474 """Try static fetch, fall back to JS rendering if needed."""
1475 from .base import ContentType
1477 if content_type is None: 1477 ↛ 1481line 1477 didn't jump to line 1481 because the condition on line 1477 was always true
1478 content_type = ContentType.TEXT
1480 # First: try static fetch (fast)
1481 logger.debug(
1482 "auto_html.static_fetch_started url={url}",
1483 url=redact_url_for_log(url),
1484 )
1485 result = super().download(url, content_type)
1487 if result and len(result) >= self.min_content_length:
1488 logger.debug(
1489 "auto_html.static_fetch_succeeded url={url} content_length={content_length}",
1490 url=redact_url_for_log(url),
1491 content_length=len(result),
1492 )
1493 return result
1495 # Host-level budget: a static fetch to this host gave up on
1496 # 429/503, and the browser leg's own rate limiting never sees those
1497 # outcomes — do not send it to a host that asked us to stop.
1498 if self._host_rate_limited(url):
1499 logger.debug(
1500 "auto_html.host_rate_limited_js_fallback_skipped url={url}",
1501 url=redact_url_for_log(url),
1502 )
1503 return result
1505 # Check if we should retry with JS rendering
1506 raw_html = getattr(self, "_last_raw_html", None)
1507 needs_js = raw_html and self._has_spa_signals(raw_html)
1508 no_content = result is None or len(result) < self.min_content_length
1510 if needs_js or no_content: 1510 ↛ 1538line 1510 didn't jump to line 1538 because the condition on line 1510 was always true
1511 if not _browser_transport_allowed(self.session):
1512 logger.debug("auto_html.js_fallback_scope_refused")
1513 return result
1514 if not self.enable_js_rendering:
1515 logger.debug(
1516 "auto_html.js_fallback_disabled url={url}",
1517 url=redact_url_for_log(url),
1518 )
1519 return result
1520 logger.info(
1521 "auto_html.js_fallback_started url={url}",
1522 url=redact_url_for_log(url),
1523 )
1524 pw_dl = self._get_playwright_downloader()
1525 pw_result = pw_dl.download(url, content_type)
1526 if pw_result and len(pw_result) > len(result or b""): 1526 ↛ 1533line 1526 didn't jump to line 1533 because the condition on line 1526 was always true
1527 logger.info(
1528 "auto_html.js_succeeded url={url} content_length={content_length}",
1529 url=redact_url_for_log(url),
1530 content_length=len(pw_result),
1531 )
1532 return pw_result
1533 logger.debug(
1534 "auto_html.js_not_improved url={url}",
1535 url=redact_url_for_log(url),
1536 )
1538 return result
1540 def download_with_result(self, url, content_type=None):
1541 """Try static fetch, fall back to JS rendering if needed."""
1542 from .base import ContentType, DownloadResult
1544 if content_type is None:
1545 content_type = ContentType.TEXT
1547 # The host mark outlives this call, this flag does not: clear it
1548 # before anything can set it, so it can only describe this URL.
1549 self._static_fetch_rate_limited = False
1551 # First: try static fetch (fast)
1552 logger.debug(
1553 "auto_html.static_fetch_started url={url}",
1554 url=redact_url_for_log(url),
1555 )
1556 result = super().download_with_result(url, content_type)
1558 if (
1559 result.is_success
1560 and result.content
1561 and len(result.content) >= self.min_content_length
1562 ):
1563 logger.debug(
1564 "auto_html.static_fetch_succeeded url={url} content_length={content_length}",
1565 url=redact_url_for_log(url),
1566 content_length=len(result.content),
1567 )
1568 return result
1570 # Host-level budget: a static fetch to this host gave up on
1571 # 429/503, and the browser leg's own rate limiting never sees those
1572 # outcomes — do not send it to a host that asked us to stop. Only
1573 # the fallback legs are skipped, as in download().
1574 if self._host_rate_limited(url):
1575 logger.debug(
1576 "auto_html.host_rate_limited_js_fallback_skipped url={url}",
1577 url=redact_url_for_log(url),
1578 )
1579 if self._static_fetch_rate_limited:
1580 # This URL's own static fetch gave up on 429/503: say so in
1581 # place of the generic fetch failure.
1582 return DownloadResult(
1583 skip_reason=(
1584 "Host rate-limited the static fetch (HTTP 429/503 "
1585 "after retries); skipping fallback legs"
1586 )
1587 )
1588 # An earlier URL marked the host. That says nothing about this
1589 # URL, so it keeps its own static result, success or failure.
1590 return result
1592 # Check if we should retry with JS rendering
1593 raw_html = getattr(self, "_last_raw_html", None)
1594 needs_js = raw_html and self._has_spa_signals(raw_html)
1595 no_content = (
1596 not result.is_success
1597 or not result.content
1598 or len(result.content) < self.min_content_length
1599 )
1601 if needs_js or no_content: 1601 ↛ 1633line 1601 didn't jump to line 1633 because the condition on line 1601 was always true
1602 if not _browser_transport_allowed(self.session):
1603 logger.debug("auto_html.js_fallback_scope_refused")
1604 return result
1605 if not self.enable_js_rendering: 1605 ↛ 1611line 1605 didn't jump to line 1611 because the condition on line 1605 was always true
1606 logger.debug(
1607 "auto_html.js_fallback_disabled url={url}",
1608 url=redact_url_for_log(url),
1609 )
1610 return result
1611 logger.info(
1612 "auto_html.js_fallback_started url={url}",
1613 url=redact_url_for_log(url),
1614 )
1615 pw_dl = self._get_playwright_downloader()
1616 pw_result = pw_dl.download_with_result(url, content_type)
1617 if (
1618 pw_result.is_success
1619 and pw_result.content
1620 and len(pw_result.content) > len(result.content or b"")
1621 ):
1622 logger.info(
1623 "auto_html.js_succeeded url={url} content_length={content_length}",
1624 url=redact_url_for_log(url),
1625 content_length=len(pw_result.content),
1626 )
1627 return pw_result
1628 logger.debug(
1629 "auto_html.js_not_improved url={url}",
1630 url=redact_url_for_log(url),
1631 )
1633 return result
1635 def close(self):
1636 """Clean up both static and JS rendering resources."""
1637 if self._playwright_downloader:
1638 self._playwright_downloader.close()
1639 self._playwright_downloader = None
1640 super().close()