Coverage for src/local_deep_research/research_library/downloaders/playwright_html.py: 85%

1""" 

2HTML Downloader with JavaScript rendering support. 

3 

4Uses Crawl4AI (default) or plain Playwright for JS-rendered pages. 

5Crawl4AI adds: robots.txt checking, shadow DOM flattening, iframe 

6inlining, smart scrolling for lazy-loaded content, and caching. 

7Falls back to plain Playwright if Crawl4AI is not installed. 

8 

9No stealth/anti-detection features are used — the browser identifies 

10honestly via BROWSER_USER_AGENT and respects robots.txt. 

11""" 

12 

13import asyncio 

14import functools 

15from datetime import datetime, timezone 

16from email.utils import parsedate_to_datetime 

17from typing import Optional 

18from urllib.parse import urljoin, urlparse 

19 

20from loguru import logger 

21 

22from .html import HTMLDownloader 

23from ...constants import BROWSER_USER_AGENT 

24from ...security import ssrf_validator 

25from ...security.safe_requests import ( 

26 _MAX_REDIRECTS, 

27 _REDIRECT_STATUS_CODES, 

28 _resolve_redirect_method, 

29) 

30 

31# Heavy subresource types dropped when ``block_resources`` is set, mirroring 

32# the previous extension-glob resource route but keyed on Playwright's 

33# resource_type (robust to query strings / extensionless URLs). 

34_BLOCKED_RESOURCE_TYPES = frozenset({"image", "media", "font", "stylesheet"}) 

35 

36# Non-http(s) URL schemes the browser may safely handle itself (they never 

37# touch the network, so they are not SSRF / local-resource vectors): the route 

38# guard lets ONLY these fall through via ``route.fallback()``. EVERY other 

39# non-http(s) scheme — ``file:`` (local file read), ``chrome:``/``chrome- 

40# extension:``, ``gopher:``, ``ftp:``, ``ws(s):`` handled elsewhere, etc. — is 

41# aborted fail-closed rather than blanket-allowed, so a redirect (or a page 

42# resource) cannot pivot the browser to a local file or an internal handler. 

43_FALLTHROUGH_NON_HTTP_SCHEMES = frozenset({"data", "blob", "about"}) 

44 

45# Neutralise Service Workers on browser paths that cannot set 

46# service_workers="block" at context creation (Crawl4AI's BrowserConfig 

47# exposes no such option). Requests made BY a Service Worker are not 

48# intercepted by page/context route handlers, so a SW could otherwise 

49# fetch an internal/metadata host unvalidated. Preventing registration 

50# forces all page traffic back through the guarded normal network path. 

51# Injected as an init script so it runs before any page script. 

52_DISABLE_SERVICE_WORKERS_JS = """ 

53try { 

54 if (window.navigator && window.navigator.serviceWorker) { 

55 const reject = () => 

56 Promise.reject(new Error('service workers are disabled')); 

57 try { window.navigator.serviceWorker.register = reject; } catch (e) {} 

58 try { 

59 Object.defineProperty(window.navigator, 'serviceWorker', { 

60 configurable: true, 

61 get() { return undefined; }, 

62 }); 

63 } catch (e) {} 

64 } 

65} catch (e) {} 

66""" 

67 

68 

69# SSRF hardening: WebRTC (RTCPeerConnection ICE / STUN / TURN) opens raw 

70# UDP/TCP sockets to an arbitrary host:port through Chromium's WebRTC stack — 

71# a data path that neither page.route() (Fetch) nor route_web_socket() 

72# instruments, so it bypasses ssrf_validator entirely. Text extraction never 

73# needs WebRTC, so neutralize the constructors before any page script runs 

74# (same fail-closed reasoning as the service-worker block). 

75_DISABLE_WEBRTC_JS = """ 

76(() => { 

77 const deny = function () { 

78 throw new DOMException('WebRTC is disabled', 'NotSupportedError'); 

79 }; 

80 for (const name of ['RTCPeerConnection', 'webkitRTCPeerConnection', 

81 'mozRTCPeerConnection', 'RTCDataChannel']) { 

82 try { 

83 Object.defineProperty(window, name, { 

84 configurable: false, enumerable: false, get() { return deny; }, 

85 }); 

86 } catch (e) {} 

87 } 

88})(); 

89""" 

90 

91 

92# Signals that a page is a JS-rendered SPA and needs browser rendering 

93SPA_SIGNALS = [ 

94 'id="root"', 

95 'id="app"', 

96 'id="__next"', 

97 "__NEXT_DATA__", 

98 "data-reactroot", 

99 'ng-version="', 

100 "<noscript>You need to enable JavaScript", 

101 "<noscript>Please enable JavaScript", 

102 "window.__INITIAL_STATE__", 

103] 

104 

105 

106def _cookie_domain_matches(host: str, cookie_domain: str) -> bool: 

107 """Whether a ``Set-Cookie`` ``Domain=`` attribute may be honored for a 

108 response served by ``host``. 

109 

110 Mirrors RFC 6265 §5.1.3 domain-matching (and real-browser behavior): the 

111 host must equal the cookie domain or be a subdomain of it. A leading dot 

112 on the attribute is ignored. Returns False for a Domain that points at a 

113 DIFFERENT or unrelated registrable domain — a cookie a browser drops — 

114 so the caller can fall back to host-scoping instead of letting a redirect 

115 hop plant a cookie on someone else's domain (session fixation). 

116 

117 Note: this does NOT consult the Public Suffix List, so a suffix like 

118 ``Domain=co.uk`` set by ``a.co.uk`` is honored here and could be re-sent 

119 to a sibling host later in the SAME redirect chain. The residual is very 

120 narrow — the browser context is ephemeral (one fetch) and Chromium 

121 enforces the PSL on its own jar — and it is strictly better than the 

122 pre-fix behavior, which forwarded all cookies cross-domain. Wiring in a 

123 PSL check would tighten it further (optional follow-up). 

124 """ 

125 if not host or not cookie_domain: 125 ↛ 126line 125 didn't jump to line 126 because the condition on line 125 was never true

126 return False 

127 host = host.lower().rstrip(".") 

128 cookie_domain = cookie_domain.lower().lstrip(".").rstrip(".") 

129 if not cookie_domain: 129 ↛ 130line 129 didn't jump to line 130 because the condition on line 129 was never true

130 return False 

131 return host == cookie_domain or host.endswith("." + cookie_domain) 

132 

133 

134def _same_registrable_site(host_a: str, host_b: str) -> bool: 

135 """Whether two request hosts belong to the same domain family for the 

136 purpose of forwarding credentialed request headers across a redirect hop. 

137 

138 Mirrors the host/subdomain relation ``_cookie_domain_matches`` already 

139 uses (RFC 6265 §5.1.3 style): equal hosts, or one a subdomain of the 

140 other, count as the SAME site; anything else is cross-site. Like that 

141 helper it does NOT consult the Public Suffix List, so it is a deliberately 

142 conservative approximation of "registrable domain" — consistent with the 

143 rest of this guard, while Chromium still enforces its own PSL on the 

144 browser jar. Used to decide when to drop ``Authorization`` / 

145 ``Proxy-Authorization`` on a cross-site hop, the twin of the per-hop 

146 ``Cookie`` re-scoping (a browser strips credentials cross-origin too). 

147 

148 Fail-CLOSED: a missing/empty host on either side returns False (treated as 

149 cross-site), so credentialed headers are dropped rather than forwarded on 

150 any ambiguity. 

151 """ 

152 if not host_a or not host_b: 152 ↛ 153line 152 didn't jump to line 153 because the condition on line 152 was never true

153 return False 

154 a = host_a.lower().rstrip(".") 

155 b = host_b.lower().rstrip(".") 

156 if not a or not b: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true

157 return False 

158 return a == b or a.endswith("." + b) or b.endswith("." + a) 

159 

160 

161def _parse_one_set_cookie(raw: str): 

162 """Parse a single ``Set-Cookie`` header value into ``(name, value, attrs)``. 

163 

164 A manual RFC 6265 §5.2 split rather than ``http.cookies.SimpleCookie``: 

165 SimpleCookie silently DROPS cookies whose name is not a strict RFC token 

166 (e.g. ``data[uid]``, which real browsers accept), so those cookies would 

167 vanish from the re-applied jar. This split keeps them. 

168 

169 Returns ``None`` when the header has no ``name=value`` pair. ``attrs`` 

170 keys are lowercased; value-less flag attributes (``Secure``, ``HttpOnly``) 

171 map to ``True``. Each ``Set-Cookie`` header carries exactly one cookie 

172 (RFC 6265 §3), and the only comma that can appear inside a value belongs 

173 to an ``Expires`` HTTP-date, which never contains a semicolon — so a 

174 plain ``;`` split is safe. 

175 """ 

176 if not raw: 176 ↛ 177line 176 didn't jump to line 177 because the condition on line 176 was never true

177 return None 

178 parts = raw.split(";") 

179 name, sep, value = parts[0].partition("=") 

180 name = name.strip() 

181 if not sep or not name: 

182 return None 

183 value = value.strip() 

184 # Strip one layer of surrounding DQUOTEs (mirrors SimpleCookie). 

185 if len(value) >= 2 and value[0] == '"' and value[-1] == '"': 185 ↛ 186line 185 didn't jump to line 186 because the condition on line 185 was never true

186 value = value[1:-1] 

187 attrs = {} 

188 for segment in parts[1:]: 

189 akey, asep, aval = segment.partition("=") 

190 akey = akey.strip().lower() 

191 if not akey: 191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true

192 continue 

193 attrs[akey] = aval.strip() if asep else True 

194 return name, value, attrs 

195 

196 

197def _set_cookie_is_expired(attrs: dict) -> bool: 

198 """Whether a parsed ``Set-Cookie`` is a deletion/expired directive that 

199 must NOT be re-applied to the jar. 

200 

201 A ``Max-Age=0`` (or negative) or a past ``Expires`` is how a server 

202 DELETES a cookie; re-adding it as a live cookie would resurrect a value 

203 the server just cleared. ``Max-Age`` takes precedence over ``Expires`` 

204 (RFC 6265 §5.3). Fail-OPEN: an unparseable attribute is treated as NOT 

205 expired, so a live cookie is never dropped by a date-parsing quirk. 

206 """ 

207 max_age = attrs.get("max-age") 

208 if isinstance(max_age, str) and max_age.strip(): 

209 try: 

210 return int(max_age) <= 0 

211 except ValueError: 

212 pass # Unparseable Max-Age -> fall through to Expires. 

213 expires = attrs.get("expires") 

214 if isinstance(expires, str) and expires.strip(): 

215 try: 

216 dt = parsedate_to_datetime(expires) 

217 if dt is None: 217 ↛ 218line 217 didn't jump to line 218 because the condition on line 217 was never true

218 return False 

219 if dt.tzinfo is None: 219 ↛ 220line 219 didn't jump to line 220 because the condition on line 219 was never true

220 dt = dt.replace(tzinfo=timezone.utc) 

221 return dt <= datetime.now(timezone.utc) 

222 except Exception: 

223 return False 

224 return False 

225 

226 

227def _run_async(coro, timeout: float = None): 

228 """Run an async coroutine from synchronous code. 

229 

230 Handles the case where an event loop is already running 

231 (e.g. inside Jupyter or an async framework) by creating 

232 a new thread with its own loop. 

233 

234 Args: 

235 coro: The coroutine to run. 

236 timeout: Max seconds to wait for the result. Prevents 

237 indefinite hangs if the coroutine's internal timeout fails. 

238 """ 

239 try: 

240 loop = asyncio.get_running_loop() 

241 except RuntimeError: 

242 loop = None 

243 

244 if loop is None: 

245 return asyncio.run(coro) 

246 

247 # Already inside an event loop — run in a new thread 

248 import concurrent.futures 

249 

250 with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: 

251 future = pool.submit(asyncio.run, coro) 

252 return future.result(timeout=timeout) 

253 

254 

255class PlaywrightHTMLDownloader(HTMLDownloader): 

256 """HTML downloader with JS rendering via Crawl4AI or Playwright. 

257 

258 Default: Crawl4AI (robots.txt, shadow DOM, iframes, caching). 

259 Fallback: plain Playwright if Crawl4AI is not installed. 

260 

261 No stealth or anti-detection features are used. 

262 """ 

263 

264 def __init__( 

265 self, 

266 timeout: int = 30, 

267 language: str = "English", 

268 wait_until: str = "networkidle", 

269 block_resources: bool = True, 

270 allow_private_ips: bool = False, 

271 **kwargs, 

272 ): 

273 super().__init__(timeout=timeout, language=language) 

274 self.wait_until = wait_until 

275 self.block_resources = block_resources 

276 # SSRF policy for the browser route guards (initial navigation + 

277 # every redirect hop + subresources). Defaults strict; relaxed to 

278 # True only under the PRIVATE_ONLY egress scope so local lab hosts 

279 # remain reachable. Cloud-metadata IPs stay blocked regardless 

280 # (``ssrf_validator`` always rejects them). Mirrors the static 

281 # SafeSession path's ``allow_private_ips`` handling. 

282 self.allow_private_ips = allow_private_ips 

283 # The guarded robots.txt check (below) reuses this SafeSession, whose 

284 # send() SSRF-validates every hop; keep its policy in lockstep with 

285 # the browser guard so both honour the same egress scope. 

286 if hasattr(self.session, "allow_private_ips"): 286 ↛ 289line 286 didn't jump to line 289 because the condition on line 286 was always true

287 self.session.allow_private_ips = allow_private_ips 

288 # Plain Playwright fallback state 

289 self._playwright = None 

290 self._browser = None 

291 

292 # ------------------------------------------------------------------ 

293 # Per-request egress guard for the JS-render browser paths. 

294 # 

295 # WHY a manual redirect loop instead of "validate the request URL and 

296 # continue": Playwright does NOT re-fire ``page.route`` handlers for the 

297 # hops of a server-side redirect chain — the browser follows 3xx 

298 # responses internally, below the interception layer. So a 

299 # ``route.continue_()`` / ``route.fallback()`` on a public URL that 

300 # 302-redirects to a cloud-metadata endpoint (link-local) or an RFC1918 

301 # host would let the browser reach the internal target unchecked — 

302 # exactly the gap the static ``SafeSession`` path already closes by 

303 # validating every hop. To reach parity we follow the redirect chain 

304 # ourselves with ``route.fetch(max_redirects=0)``, validate EACH hop's 

305 # target before fetching it, and hand the browser only the final, safe 

306 # response via ``route.fulfill``. The browser therefore never issues its 

307 # own redirect-following request to an unvalidated destination. 

308 # ------------------------------------------------------------------ 

309 

310 def _playwright_route_guard(self, route) -> None: 

311 """Egress guard for the plain-Playwright (sync) browser path. 

312 

313 Validates the initial navigation and every redirect hop against 

314 ``ssrf_validator.validate_url`` before the browser connects, and 

315 drops heavy subresources when ``block_resources`` is set. Also 

316 re-scopes cookies per redirect hop (via ``_reapply_hop_cookies_sync``) 

317 so a cookie set on one hop is never forwarded to a different 

318 registrable domain a later hop redirects to — the same protection the 

319 crawl4ai path applies via ``_reapply_hop_cookies``. 

320 """ 

321 request = route.request 

322 first_url = request.url 

323 scheme = urlparse(first_url).scheme.lower() 

324 if scheme not in ("http", "https"): 

325 # ALLOWLIST, fail-closed: only data:/blob:/about: (never touch the 

326 # network) may fall through to the browser. Every other non-http(s) 

327 # scheme — file: (local file read), chrome:, gopher:, ftp: … — is 

328 # aborted so a redirect or page resource cannot pivot the browser 

329 # to a local file or an internal handler. 

330 if scheme in _FALLTHROUGH_NON_HTTP_SCHEMES: 

331 route.fallback() 

332 else: 

333 route.abort() 

334 return 

335 if ( 

336 self.block_resources 

337 and request.resource_type in _BLOCKED_RESOURCE_TYPES 

338 ): 

339 route.abort() 

340 return 

341 

342 current = first_url 

343 method = request.method 

344 # None => reuse the intercepted request's body; "" => drop the body 

345 # after a POST->GET redirect downgrade (parity with safe_post). 

346 body_override = None 

347 # Per-hop "Cookie" header, re-derived from the browser context's 

348 # cookie jar (scoped to the NEXT hop's URL) on EVERY hop by 

349 # _reapply_hop_cookies_sync. It stays None only for the very first 

350 # fetch, which legitimately uses the originally intercepted request's 

351 # own Cookie header; from the first redirect onward it is always set 

352 # (possibly to an explicit EMPTY header). This is required because 

353 # route.fetch() otherwise keeps resending the ORIGINAL intercepted 

354 # request's Cookie header for the rest of this handler invocation — 

355 # so without a per-hop override a cookie scoped to hop N's domain 

356 # would ride along to a cross-host hop N+1 (a cross-origin cookie 

357 # leak). Mirrors the crawl4ai path's _reapply_hop_cookies. 

358 cookie_header_override = None 

359 for _ in range(_MAX_REDIRECTS + 1): 

360 if not ssrf_validator.validate_url( 

361 current, allow_private_ips=self.allow_private_ips 

362 ): 

363 # Log scheme://host:port only — a request URL may carry 

364 # credentials in userinfo / query params (RFC 3986 §3.2.1). 

365 logger.warning( 

366 "Playwright: blocked SSRF-unsafe browser request to {}", 

367 ssrf_validator.redact_url_for_log(current), 

368 ) 

369 route.abort("blockedbyclient") 

370 return 

371 fetch_kwargs = { 

372 "url": current, 

373 "method": method, 

374 "max_redirects": 0, 

375 } 

376 if body_override is not None: 

377 fetch_kwargs["post_data"] = body_override 

378 try: 

379 self._apply_hop_cookie_header( 

380 request, fetch_kwargs, cookie_header_override 

381 ) 

382 except Exception: 

383 # Fail CLOSED: building the hop headers reads 

384 # request.headers and must never let an exception escape 

385 # this route handler and fall through to an unguarded 

386 # fetch — abort the hop exactly like a failed fetch below. 

387 logger.debug( 

388 "Playwright: failed to build guarded hop headers", 

389 exc_info=True, 

390 ) 

391 route.abort("failed") 

392 return 

393 try: 

394 response = route.fetch(**fetch_kwargs) 

395 except Exception: 

396 logger.debug("Playwright: guarded fetch failed", exc_info=True) 

397 route.abort("failed") 

398 return 

399 next_url = self._redirect_target(current, response) 

400 # Re-apply this hop's Set-Cookie header(s) to the context AND 

401 # re-derive, from the context jar scoped to the NEXT hop, the 

402 # explicit Cookie header for that hop's route.fetch() — BEFORE 

403 # following it. next_url is None on the terminal hop (no next 

404 # request needs a header). See _reapply_hop_cookies for why the 

405 # per-hop re-derivation is required. 

406 cookie_header_override = self._reapply_hop_cookies_sync( 

407 route, 

408 current, 

409 response, 

410 next_url, 

411 ) 

412 if next_url is None: 

413 route.fulfill(response=response) 

414 return 

415 new_method = _resolve_redirect_method(method, response.status) 

416 if new_method == "GET" and method != "GET": 

417 body_override = "" # POST->GET downgrade must not leak body 

418 method = new_method 

419 current = next_url 

420 logger.warning( 

421 "Playwright: too many redirects — blocking browser request" 

422 ) 

423 route.abort("blockedbyclient") 

424 

425 def _playwright_ws_guard(self, ws_route) -> None: 

426 """Block every WebSocket on the plain-Playwright path. 

427 

428 ``route("**/*")`` does not match WS upgrades, so page JS could open 

429 ``ws(s)://<internal>`` unvalidated. WebSockets are not needed for 

430 HTML text extraction, so reject all of them: simply returning 

431 WITHOUT calling ``connect_to_server`` means the connection to the 

432 real server is never established (verified: the target sees zero 

433 connections). We deliberately do NOT call ``ws_route.close()`` here — 

434 in Playwright's SYNC API a blocking route-handler call re-enters the 

435 dispatcher thread that ``page.goto`` is parked on and DEADLOCKS; the 

436 no-op form blocks the server just as effectively without that risk. 

437 """ 

438 try: 

439 logger.debug( 

440 "Playwright: blocking WebSocket to {}", 

441 ssrf_validator.redact_url_for_log(ws_route.url), 

442 ) 

443 except Exception: 

444 logger.debug("Playwright: blocking WebSocket") 

445 

446 async def _crawl4ai_route_guard(self, route) -> None: 

447 """Egress guard for the Crawl4AI (async Playwright) browser path. 

448 

449 Same policy as ``_playwright_route_guard`` but async: the blocking 

450 DNS resolution inside ``validate_url`` is offloaded to a thread so it 

451 does not stall Crawl4AI's event loop. Installed on each page context 

452 via the ``on_page_context_created`` hook. 

453 """ 

454 request = route.request 

455 first_url = request.url 

456 scheme = urlparse(first_url).scheme.lower() 

457 if scheme not in ("http", "https"): 

458 # ALLOWLIST, fail-closed (see _playwright_route_guard): only 

459 # data:/blob:/about: fall through; every other non-http(s) scheme 

460 # (file:/chrome:/gopher: …) is aborted so it cannot pivot the 

461 # browser to a local file or an internal handler. 

462 if scheme in _FALLTHROUGH_NON_HTTP_SCHEMES: 

463 await route.fallback() 

464 else: 

465 await route.abort() 

466 return 

467 if ( 

468 self.block_resources 

469 and request.resource_type in _BLOCKED_RESOURCE_TYPES 

470 ): 

471 await route.abort() 

472 return 

473 

474 loop = asyncio.get_running_loop() 

475 current = first_url 

476 method = request.method 

477 body_override = None 

478 # Carries the "Cookie" header for the NEXT hop, re-derived from the 

479 # context's cookie jar (scoped to that hop's URL) on EVERY hop by 

480 # _reapply_hop_cookies. It stays None only for the very first fetch, 

481 # which legitimately uses the originally intercepted request's own 

482 # Cookie header; from the first redirect onward it is always set 

483 # (possibly to an explicit EMPTY header). This is required because 

484 # route.fetch() otherwise keeps resending the ORIGINAL intercepted 

485 # request's Cookie header for the rest of this handler invocation — 

486 # so without a per-hop override a cookie scoped to hop N's domain 

487 # would ride along to hop N+1 even across a DIFFERENT registrable 

488 # domain (a cross-origin cookie leak). 

489 cookie_header_override = None 

490 for _ in range(_MAX_REDIRECTS + 1): 490 ↛ 556line 490 didn't jump to line 556 because the loop on line 490 didn't complete

491 ok = await loop.run_in_executor( 

492 None, 

493 functools.partial( 

494 ssrf_validator.validate_url, 

495 current, 

496 allow_private_ips=self.allow_private_ips, 

497 ), 

498 ) 

499 if not ok: 

500 logger.warning( 

501 "Crawl4AI: blocked SSRF-unsafe browser request to {}", 

502 ssrf_validator.redact_url_for_log(current), 

503 ) 

504 await route.abort("blockedbyclient") 

505 return 

506 fetch_kwargs = { 

507 "url": current, 

508 "method": method, 

509 "max_redirects": 0, 

510 } 

511 if body_override is not None: 

512 fetch_kwargs["post_data"] = body_override 

513 try: 

514 self._apply_hop_cookie_header( 

515 request, fetch_kwargs, cookie_header_override 

516 ) 

517 except Exception: 

518 # Fail CLOSED: building the hop headers reads 

519 # request.headers and must never let an exception escape 

520 # this route handler and fall through to an unguarded 

521 # fetch — abort the hop exactly like a failed fetch below. 

522 logger.debug( 

523 "Crawl4AI: failed to build guarded hop headers", 

524 exc_info=True, 

525 ) 

526 await route.abort("failed") 

527 return 

528 try: 

529 response = await route.fetch(**fetch_kwargs) 

530 except Exception: 

531 logger.debug("Crawl4AI: guarded fetch failed", exc_info=True) 

532 await route.abort("failed") 

533 return 

534 next_url = self._redirect_target(current, response) 

535 # Re-apply this hop's Set-Cookie header(s) to the context AND 

536 # re-derive, from the context jar scoped to the NEXT hop, the 

537 # explicit Cookie header for that hop's route.fetch() — BEFORE 

538 # following it. Passing next_url (which is None on the terminal 

539 # hop) lets _reapply_hop_cookies skip deriving a header when 

540 # there is no next request. See _reapply_hop_cookies for why the 

541 # per-hop re-derivation is required on the crawl4ai path. 

542 cookie_header_override = await self._reapply_hop_cookies( 

543 route, 

544 current, 

545 response, 

546 next_url, 

547 ) 

548 if next_url is None: 

549 await route.fulfill(response=response) 

550 return 

551 new_method = _resolve_redirect_method(method, response.status) 

552 if new_method == "GET" and method != "GET": 

553 body_override = "" # POST->GET downgrade must not leak body 

554 method = new_method 

555 current = next_url 

556 logger.warning( 

557 "Crawl4AI: too many redirects — blocking browser request" 

558 ) 

559 await route.abort("blockedbyclient") 

560 

561 async def _crawl4ai_ws_guard(self, ws_route) -> None: 

562 """Block every WebSocket on the Crawl4AI path (async twin of 

563 ``_playwright_ws_guard``). 

564 

565 Same no-op-reject strategy: not calling ``connect_to_server`` leaves 

566 the real server unreached. Kept no-op (no ``close()``) for symmetry 

567 with the sync guard and to avoid any handler-reentrancy surprises. 

568 """ 

569 try: 

570 logger.debug( 

571 "Crawl4AI: blocking WebSocket to {}", 

572 ssrf_validator.redact_url_for_log(ws_route.url), 

573 ) 

574 except Exception: 

575 logger.debug("Crawl4AI: blocking WebSocket") 

576 

577 @staticmethod 

578 def _cookies_from_response(response, response_url: str) -> list: 

579 """Parse ``Set-Cookie`` response header(s) into Playwright cookie 

580 dicts, ready for ``BrowserContext.add_cookies()``. 

581 

582 WHY THIS EXISTS: both browser guards re-derive the ``Cookie`` request 

583 header per redirect hop from the context's own jar, so this parses a 

584 hop's ``Set-Cookie`` header(s) and re-applies them to the jar before 

585 the next hop is followed. On the crawl4ai path it is also load-bearing 

586 for content fidelity: crawl4ai pre-seeds the page context with a 

587 marker cookie via ``context.add_cookies()`` before navigation 

588 (``BrowserManager.setup_context``), and once a context's jar has been 

589 touched that way, Playwright's ``route.fetch()`` on a subsequent 

590 redirect hop does NOT merge that hop's ``Set-Cookie`` into the jar for 

591 us — so a session/auth cookie set on an intermediate hop would 

592 silently be lost, corrupting the final page (e.g. a login- or 

593 region-gated redirect). 

594 

595 A ``Set-Cookie`` that is really a DELETION directive (``Max-Age=0`` or 

596 a past ``Expires``) is honored and NOT re-added, so a cookie the 

597 server just cleared is never resurrected onto the jar. 

598 

599 Uses ``headers_array`` (a list of ``{name, value}`` pairs) rather 

600 than the ``headers`` dict, because a response can carry MULTIPLE 

601 ``Set-Cookie`` headers — a plain dict would silently collapse them 

602 to one. 

603 """ 

604 host = urlparse(response_url).hostname 

605 if not host: 605 ↛ 606line 605 didn't jump to line 606 because the condition on line 605 was never true

606 return [] 

607 try: 

608 headers = response.headers_array 

609 except Exception: 

610 return [] 

611 cookies = [] 

612 for header in headers: 

613 if str(header.get("name", "")).lower() != "set-cookie": 

614 continue 

615 raw = header.get("value", "") 

616 try: 

617 parsed = _parse_one_set_cookie(raw) 

618 except Exception: 

619 logger.debug( 

620 "Browser guard: failed to parse a Set-Cookie header", 

621 exc_info=True, 

622 ) 

623 continue 

624 if parsed is None: 

625 continue 

626 name, value, attrs = parsed 

627 if _set_cookie_is_expired(attrs): 

628 # A Max-Age=0 / past-Expires Set-Cookie is a DELETION 

629 # directive: re-adding it would resurrect a cookie the 

630 # server just cleared. Honor the expiry and skip it. 

631 logger.debug( 

632 "Browser guard: skipping expired/deletion Set-Cookie" 

633 ) 

634 continue 

635 raw_domain = attrs.get("domain") 

636 if ( 

637 isinstance(raw_domain, str) 

638 and raw_domain 

639 and _cookie_domain_matches(host, raw_domain) 

640 ): 

641 cookie_domain = raw_domain 

642 else: 

643 # No Domain attribute, or a Domain that does NOT 

644 # domain-match the response host (a cross-domain or 

645 # unrelated Domain a browser rejects). Scope the cookie 

646 # to the response host — mirroring browser behavior and 

647 # preventing a hop from planting a cookie on another 

648 # registrable domain (session fixation). 

649 if isinstance(raw_domain, str) and raw_domain: 

650 logger.debug( 

651 "Browser guard: rejecting a Set-Cookie Domain that " 

652 "does not match the response host; scoping the " 

653 "cookie to the host instead" 

654 ) 

655 cookie_domain = host 

656 path = attrs.get("path") 

657 cookie = { 

658 "name": name, 

659 "value": value, 

660 "domain": cookie_domain, 

661 "path": path if isinstance(path, str) and path else "/", 

662 } 

663 if attrs.get("secure"): 

664 cookie["secure"] = True 

665 if attrs.get("httponly"): 

666 cookie["httpOnly"] = True 

667 cookies.append(cookie) 

668 return cookies 

669 

670 async def _reapply_hop_cookies( 

671 self, 

672 route, 

673 current_url: str, 

674 response, 

675 next_hop_url: Optional[str], 

676 ) -> Optional[str]: 

677 """Apply ``response``'s ``Set-Cookie`` header(s) to the browser 

678 context and return the ``Cookie`` header the caller must pass to the 

679 NEXT hop's ``route.fetch(headers=...)`` — re-derived, on EVERY hop, 

680 from the context jar scoped to ``next_hop_url``. 

681 

682 Why re-derive on every hop instead of carrying the previous hop's 

683 header forward: ``route.fetch()`` keeps resending the ORIGINAL 

684 intercepted request's Cookie header for the rest of this guard 

685 invocation unless we override it. If we ever carried a previous 

686 hop's Cookie header (or let the original one ride along) across a 

687 redirect to a DIFFERENT registrable domain, that domain would 

688 receive cookies scoped to the previous one — a cross-origin cookie 

689 leak a real browser never performs. So we ALWAYS read the jar back 

690 through ``context.cookies(next_hop_url)`` (Playwright scopes it by 

691 domain/path) and return exactly those cookies: 

692 

693 * same-domain next hop -> the jar returns that domain's cookies 

694 (including any just set on this hop, applied below via 

695 ``add_cookies``), so same-domain persistence keeps working; 

696 * cross-domain next hop -> the jar returns nothing for that domain, 

697 so we return an EMPTY header and the caller sends NO Cookie to it. 

698 

699 ``context.add_cookies()`` still runs for its own sake: it keeps the 

700 jar correct for anything reading it independently of this walk 

701 (subresources of the eventually-fulfilled page, ``document.cookie``, 

702 later navigations in the same context). 

703 

704 Fail-SAFE: on any parse/apply/read error we return an EMPTY header 

705 rather than a previous hop's, so an error can only DROP a cookie, 

706 never leak one cross-domain. This never affects the SSRF 

707 validate/abort decisions around it. Returns ``None`` only for the 

708 terminal hop (``next_hop_url is None`` — no next request needs a 

709 header); that value is unused by the caller. 

710 """ 

711 try: 

712 context = route.request.frame.page.context 

713 cookies = self._cookies_from_response(response, current_url) 

714 if cookies: 

715 await context.add_cookies(cookies) 

716 if next_hop_url is None: 

717 # Terminal hop: no next request, so no Cookie header to 

718 # derive. The jar was still updated above. 

719 return None 

720 live_cookies = await context.cookies(next_hop_url) 

721 return "; ".join(f"{c['name']}={c['value']}" for c in live_cookies) 

722 except Exception: 

723 logger.debug( 

724 "Crawl4AI: failed to re-apply redirect-hop cookies", 

725 exc_info=True, 

726 ) 

727 # Prefer sending NO cookie to the next hop over resending a 

728 # previous hop's (possibly cross-domain) one: "" clears the 

729 # Cookie header. None is used only when there is no next hop. 

730 return "" if next_hop_url is not None else None 

731 

732 def _reapply_hop_cookies_sync( 

733 self, 

734 route, 

735 current_url: str, 

736 response, 

737 next_hop_url: Optional[str], 

738 ) -> Optional[str]: 

739 """Synchronous twin of ``_reapply_hop_cookies`` for the plain 

740 (sync-API) Playwright path. 

741 

742 Same contract and fail-SAFE behavior — see ``_reapply_hop_cookies`` 

743 for the full rationale. Applies this hop's ``Set-Cookie`` header(s) 

744 to the browser context and returns the ``Cookie`` header for the NEXT 

745 hop, re-derived from the context jar scoped to ``next_hop_url`` so a 

746 cookie belonging to one domain is never forwarded to a different 

747 registrable domain a later redirect lands on. Returns ``None`` only 

748 for the terminal hop; on any error returns an EMPTY header (drop a 

749 cookie, never leak one cross-domain). 

750 """ 

751 try: 

752 context = route.request.frame.page.context 

753 cookies = self._cookies_from_response(response, current_url) 

754 if cookies: 

755 context.add_cookies(cookies) 

756 if next_hop_url is None: 

757 # Terminal hop: no next request, so no Cookie header to 

758 # derive. The jar was still updated above. 

759 return None 

760 live_cookies = context.cookies(next_hop_url) 

761 return "; ".join(f"{c['name']}={c['value']}" for c in live_cookies) 

762 except Exception: 

763 logger.debug( 

764 "Playwright: failed to re-apply redirect-hop cookies", 

765 exc_info=True, 

766 ) 

767 return "" if next_hop_url is not None else None 

768 

769 @staticmethod 

770 def _apply_hop_cookie_header( 

771 request, fetch_kwargs: dict, cookie_header_override: Optional[str] 

772 ) -> None: 

773 """Override ONLY the ``Cookie`` header on ``fetch_kwargs`` with the 

774 per-hop re-derived ``cookie_header_override``. 

775 

776 Shared by both browser guards (sync + crawl4ai). Rebuilds the base 

777 headers ``route.fetch()`` would otherwise default to (the original 

778 intercepted request's) MINUS its ``Cookie`` header, then sets the 

779 freshly re-derived Cookie header. When that value is EMPTY, the Cookie 

780 header is dropped entirely rather than sent empty: the next hop's 

781 domain has no cookies in the jar, so sending nothing is correct — and, 

782 crucially, this stops ``route.fetch()`` from resending the ORIGINAL 

783 intercepted request's Cookie header to what may be a DIFFERENT 

784 registrable domain (the cross-origin leak this guards against). 

785 

786 The original request's ``Host`` and ``Content-Length`` are ALSO dropped 

787 from the rebuilt headers on every hop: a redirect hop's ``Host`` must be 

788 derived from the NEW target (a forwarded stale ``Host`` is wrong on a 

789 cross-host hop), and after a POST->GET downgrade (body dropped to "") 

790 the original ``Content-Length`` no longer matches the body. 

791 ``route.fetch()`` recomputes both from the hop URL and the actual body 

792 once these headers are absent. 

793 

794 On a CROSS-SITE hop the originally intercepted request's 

795 ``Authorization`` / ``Proxy-Authorization`` headers are ALSO dropped 

796 (in addition to ``Cookie``): those credentials are scoped to the 

797 original origin, so — exactly as a real browser strips Authorization 

798 on a cross-origin redirect — they must never ride along to a different 

799 registrable domain a later hop lands on. Same-site hops keep them so 

800 legitimate within-site auth redirects (e.g. api.example.com -> 

801 example.com) still work. Cross-site is decided by 

802 ``_same_registrable_site`` — the same host/subdomain notion the cookie 

803 re-scoping relies on. 

804 

805 A ``None`` override (the very first hop only) leaves ``fetch_kwargs`` 

806 untouched so the initial request keeps its own Cookie header. The very 

807 first hop is by construction same-origin (its target IS the original 

808 request URL), so no credential is ever leaked by that early return. 

809 """ 

810 if cookie_header_override is None: 

811 return 

812 # Dropped on EVERY (post-first) hop, before rebuilding the headers: 

813 # * cookie — always re-derived per hop below. 

814 # * host — a redirect hop's Host must come from the NEW target, not a 

815 # stale forwarded value; a cross-host hop that kept the previous 

816 # hop's Host would send the wrong Host. ``route.fetch()`` derives the 

817 # correct Host from the hop URL once this header is absent. 

818 # * content-length — after a POST->GET redirect downgrade the body is 

819 # dropped to "" (see the guard loop's ``body_override``), so a 

820 # forwarded original Content-Length would be incorrect; 

821 # ``route.fetch()`` recomputes it from the actual (post_data) body. 

822 drop = {"cookie", "host", "content-length"} 

823 # On a cross-site hop also strip the credential headers the original 

824 # request carried (a browser strips Authorization cross-origin too). 

825 origin_host = urlparse(request.url).hostname 

826 target_host = urlparse(fetch_kwargs.get("url", "")).hostname 

827 if not _same_registrable_site(origin_host, target_host): 

828 drop |= {"authorization", "proxy-authorization"} 

829 base_headers = { 

830 k: v for k, v in request.headers.items() if k.lower() not in drop 

831 } 

832 if cookie_header_override: 

833 base_headers["cookie"] = cookie_header_override 

834 fetch_kwargs["headers"] = base_headers 

835 

836 @staticmethod 

837 def _redirect_target(current_url: str, response) -> Optional[str]: 

838 """Return the absolute redirect target for ``response``, or None. 

839 

840 None means the response is terminal (not a followed 3xx, or a 3xx 

841 with no ``Location``) and should be served to the browser as-is. 

842 Mirrors the static path's redirect handling so both validate the 

843 same hops. 

844 """ 

845 if response.status not in _REDIRECT_STATUS_CODES: 

846 return None 

847 location = response.headers.get("location") 

848 if not location: 848 ↛ 849line 848 didn't jump to line 849 because the condition on line 848 was never true

849 return None 

850 return urljoin(current_url, location.strip()) 

851 

852 def _robots_allows(self, url: str) -> bool: 

853 """Guarded robots.txt politeness check. 

854 

855 REPLACES Crawl4AI's built-in ``check_robots_txt`` fetch, which used a 

856 raw ``aiohttp`` client (redirect-following, no SSRF check) — a 

857 ``/robots.txt`` that 302-redirected to an internal/metadata host was 

858 fetched unguarded. Here the fetch goes through the downloader's 

859 ``SafeSession``, whose ``send()`` SSRF-validates every hop (initial + 

860 redirects) under the same egress scope as the browser guard. 

861 

862 Fails OPEN (returns True) on any non-200, network error, SSRF block, 

863 or parse failure — matching the "allow on robots uncertainty" 

864 convention. It only ever DISALLOWS on an explicit robots rule, so a 

865 blocked (SSRF-unsafe) robots fetch never grants extra access. 

866 """ 

867 from urllib.robotparser import RobotFileParser 

868 

869 try: 

870 parsed = urlparse(url) 

871 if not parsed.netloc: 871 ↛ 872line 871 didn't jump to line 872 because the condition on line 871 was never true

872 return True 

873 scheme = parsed.scheme or "http" 

874 robots_url = f"{scheme}://{parsed.netloc}/robots.txt" 

875 response = self.session.get( 

876 robots_url, 

877 timeout=min(self.timeout, 10), 

878 allow_redirects=True, 

879 ) 

880 if response.status_code != 200 or not response.text: 880 ↛ 881line 880 didn't jump to line 881 because the condition on line 880 was never true

881 return True 

882 parser = RobotFileParser() 

883 parser.parse(response.text.splitlines()) 

884 return parser.can_fetch(BROWSER_USER_AGENT, url) 

885 except Exception: 

886 # SafeSession raises ValueError on an SSRF-unsafe hop; network / 

887 # parse errors land here too. Fail open for politeness — the 

888 # actual navigation is still validated by the browser guard. 

889 logger.debug("Guarded robots.txt check failed open", exc_info=True) 

890 return True 

891 

892 def _fetch_html(self, url: str) -> Optional[str]: 

893 """Fetch HTML with JS rendering. 

894 

895 Tries Crawl4AI first (with robots.txt, shadow DOM, iframes), 

896 falls back to plain Playwright. 

897 """ 

898 # Entry-URL scheme guard (fail-closed). The browser downloader is 

899 # reachable via download()/download_with_result(), which do NOT 

900 # constrain the scheme themselves — only can_handle() checks it, and 

901 # not every caller consults can_handle() (e.g. 

902 # extraction/pipeline.py calls download() directly). The per-request 

903 # route guard validates redirect hops and subresources, but a 

904 # top-level page.goto()/crawler.arun() on a non-http(s) entry URL 

905 # (e.g. file:///etc/passwd) could read a LOCAL FILE and return it as 

906 # page content before any hop is walked. Refuse non-http(s) entry URLs 

907 # here so such a URL is never navigated. 

908 if urlparse(url).scheme.lower() not in ("http", "https"): 

909 logger.warning( 

910 "Browser downloader: refusing non-http(s) entry URL {}", 

911 ssrf_validator.redact_url_for_log(url), 

912 ) 

913 return None 

914 # Try Crawl4AI first (richer features, robots.txt) 

915 html = self._fetch_with_crawl4ai(url) 

916 if html is not None: 916 ↛ 923line 916 didn't jump to line 923 because the condition on line 916 was always true

917 # Crawl4AI succeeded (non-empty) or intentionally blocked 

918 # by robots.txt (empty string). Either way, don't fall 

919 # through to Playwright. 

920 return html or None 

921 

922 # Crawl4AI not installed or failed — fall back to Playwright 

923 return self._fetch_with_playwright(url) 

924 

925 def _fetch_with_crawl4ai(self, url: str) -> Optional[str]: 

926 """Fetch HTML using Crawl4AI with ethical defaults.""" 

927 domain = urlparse(url).netloc 

928 engine_type = f"crawl4ai_download_{domain}" 

929 

930 try: 

931 from crawl4ai import ( 

932 AsyncWebCrawler, 

933 BrowserConfig, 

934 CrawlerRunConfig, 

935 ) 

936 except ImportError: 

937 logger.debug("crawl4ai not installed — using Playwright") 

938 return None 

939 

940 # Ethical robots.txt check — done HERE through the guarded 

941 # SafeSession, NOT via crawl4ai's check_robots_txt (disabled below): 

942 # crawl4ai fetches robots.txt with a raw aiohttp client that follows 

943 # redirects with no SSRF validation, so a redirecting /robots.txt is 

944 # an SSRF vector that bypasses the browser route guard entirely. 

945 if not self._robots_allows(url): 

946 logger.info( 

947 "Crawl4AI: blocked by robots.txt for {}", 

948 ssrf_validator.redact_url_for_log(url), 

949 ) 

950 return "" # Empty string signals intentional skip 

951 

952 logger.debug(f"Crawl4AI fetch: {url}") 

953 wait_time = self.rate_tracker.apply_rate_limit(engine_type) 

954 

955 browser_cfg = BrowserConfig( 

956 headless=True, 

957 verbose=False, 

958 user_agent=BROWSER_USER_AGENT, 

959 ) 

960 run_cfg = CrawlerRunConfig( 

961 # robots.txt handled by the guarded _robots_allows() above; 

962 # crawl4ai's own fetch is unguarded (SSRF), so keep it OFF. 

963 check_robots_txt=False, 

964 # Better extraction: flatten modern web features 

965 flatten_shadow_dom=True, 

966 process_iframes=True, 

967 # Trigger lazy-loaded content 

968 scan_full_page=True, 

969 # Performance 

970 wait_until=self.wait_until, 

971 page_timeout=self.timeout * 1000, 

972 exclude_all_images=self.block_resources, 

973 # No stealth 

974 override_navigator=False, 

975 magic=False, 

976 simulate_user=False, 

977 verbose=False, 

978 ) 

979 

980 try: 

981 

982 async def _crawl(): 

983 async with AsyncWebCrawler(config=browser_cfg) as crawler: 

984 # Install the SSRF route guard on every page context 

985 # BEFORE any navigation. FAIL-SAFE: if the hook API is 

986 # unavailable (crawl4ai API drift), do NOT run crawl4ai 

987 # unguarded — return None so _fetch_html falls through 

988 # to the guarded plain-Playwright path. 

989 async def _on_page_context_created( 

990 page, context=None, config=None, **kwargs 

991 ): 

992 # Install ALL three guards fail-CLOSED: if any raises 

993 # (crawl4ai/Playwright API drift, or a context with 

994 # incomplete API parity), let it propagate so the 

995 # crawl is abandoned — crawl4ai's execute_hook does not 

996 # swallow it, so it reaches the outer ``except`` below 

997 # and _fetch_with_crawl4ai returns None, falling back