Coverage for src/local_deep_research/web/fastapi_app.py: 92%
760 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""
2FastAPI application factory for Local Deep Research.
4This replaces Flask's app_factory.py as the primary web application entry point.
5It mounts Socket.IO as an ASGI sub-app, configures middleware, templates,
6static files, and background services.
7"""
9import ipaddress
10import logging
11import os
12import re
13import secrets
14import sys
15import time
16from contextlib import asynccontextmanager
17from importlib import resources as importlib_resources
18from pathlib import Path
20from fastapi import FastAPI, Request
21from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
22from loguru import logger
23from starlette.middleware.sessions import SessionMiddleware
25from ..__version__ import __version__
26from ..security import get_security_default
27from ..utilities.type_utils import to_bool
28from ..utilities.log_utils import InterceptHandler
30# ---------------------------------------------------------------------------
31# Directories
32# ---------------------------------------------------------------------------
34try:
35 _PACKAGE_DIR = importlib_resources.files("local_deep_research") / "web"
36 with importlib_resources.as_file(_PACKAGE_DIR) as _pkg:
37 STATIC_DIR = (_pkg / "static").as_posix()
38 TEMPLATE_DIR = (_pkg / "templates").as_posix()
39except Exception:
40 STATIC_DIR = str(Path("static").resolve())
41 TEMPLATE_DIR = str(Path("templates").resolve())
43# Templates shared across all routers (defined in template_config to avoid circular imports)
44from .template_config import (
45 templates,
46)
48# ---------------------------------------------------------------------------
49# Secret key (same logic as Flask app_factory)
50# ---------------------------------------------------------------------------
53def _load_secret_key() -> str:
54 """Load or generate a persistent SECRET_KEY for session signing.
56 Falling back to an in-memory key on read errors silently invalidates
57 every existing session on restart and produces no operator alert,
58 so this raises a hard error instead.
59 """
60 from ..config.paths import get_data_directory
62 secret_key_file = Path(get_data_directory()) / ".secret_key"
64 from ..security.directory_creation import create_directory
66 create_directory(
67 secret_key_file.parent, context="secret key file directory"
68 )
69 new_key = secrets.token_hex(32)
70 try:
71 fd = os.open(
72 str(secret_key_file), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600
73 )
74 try:
75 os.write(fd, new_key.encode())
76 finally:
77 os.close(fd)
78 logger.info("Generated new SECRET_KEY for this installation")
79 return new_key
80 except FileExistsError:
81 # File exists — read it. Any error here is fatal: an ephemeral
82 # in-memory key would silently invalidate every existing session
83 # on restart and prevent any future restart from being consistent.
84 try:
85 with open(secret_key_file, "r", encoding="utf-8") as f:
86 key = f.read().strip()
87 except Exception as e:
88 raise RuntimeError(
89 f"Cannot read SECRET_KEY file at {secret_key_file}: {e}. "
90 "Fix the file permissions/contents or remove the file to "
91 "regenerate (this will invalidate all sessions)."
92 ) from e
93 # Empty / whitespace-only file is also fatal — itsdangerous would
94 # accept it and derive a deterministic signing key from no secret,
95 # silently breaking session integrity.
96 if not key:
97 raise RuntimeError(
98 f"SECRET_KEY file at {secret_key_file} is empty. "
99 "Delete the file to regenerate a new key (this will "
100 "invalidate all existing sessions)."
101 )
102 return key
103 except OSError as e:
104 # Cannot write the key file. Same reasoning — refuse rather than
105 # silently fall back to a non-persistent key.
106 raise RuntimeError(
107 f"Cannot write SECRET_KEY file at {secret_key_file}: {e}. "
108 "Fix the data directory permissions and retry."
109 ) from e
112SECRET_KEY = _load_secret_key()
115def warn_if_threadpool_exceeds_db_pool(worker_threads: int) -> bool:
116 """Warn when the AnyIO worker pool is larger than a user's DB pool.
118 Raising ``web.threadpool_max_threads`` above the per-user pool capacity is
119 not merely a tuning choice. ``WorkerCleanupAPIRoute`` returns each sync
120 route's connections when that request finishes, but every concurrently
121 active DB-touching worker can still hold one pool slot. More workers than
122 slots therefore permit a burst from one user to exhaust the pool before
123 any request reaches its cleanup boundary.
125 Below the pool capacity that is harmless, which is why this warns instead
126 of failing. Above it, one user's concurrent requests can exhaust their own
127 pool, after which every further request waits out ``pool_timeout``.
129 Returns True when the warning fired, so it can be tested directly rather
130 than only through a lifespan startup.
131 """
132 from ..database.pool_config import MAX_OVERFLOW, POOL_SIZE
134 pool_capacity = POOL_SIZE + MAX_OVERFLOW
135 if worker_threads <= pool_capacity:
136 return False
137 logger.warning(
138 "web.threadpool_max_threads ({}) exceeds the per-user database pool "
139 "capacity ({} = pool_size {} + max_overflow {}). Concurrent "
140 "synchronous routes can each hold one connection, so a single user's "
141 "concurrent requests can exhaust their pool and then block for "
142 "pool_timeout. Keep this at or below {}.",
143 worker_threads,
144 pool_capacity,
145 POOL_SIZE,
146 MAX_OVERFLOW,
147 pool_capacity,
148 )
149 return True
152# ---------------------------------------------------------------------------
153# Lifespan: startup / shutdown
154# ---------------------------------------------------------------------------
157@asynccontextmanager
158async def lifespan(app: FastAPI):
159 """Manage application lifecycle — start/stop background services."""
160 # --- Startup ---
161 logger.info(f"Starting Local Deep Research v{__version__}")
163 # Capture uvicorn's running event loop so background threads can
164 # dispatch Socket.IO emits via run_coroutine_threadsafe. Without
165 # this, emits from research workers / log queue silently no-op
166 # (asyncio.get_event_loop() in a worker thread doesn't return uvicorn's loop).
167 import asyncio as _asyncio
168 from .services.socketio_asgi import init_lock, set_main_loop
170 set_main_loop(_asyncio.get_running_loop())
171 # Eagerly create the Socket.IO subscription lock now that the loop is
172 # running, so the first connect/subscribe events don't race on lazy init.
173 init_lock()
175 # Size the AnyIO worker pool that serves every plain `def` route.
176 #
177 # Starlette runs sync handlers via `anyio.to_thread.run_sync`, whose
178 # default CapacityLimiter is 40 threads — and that pool is SHARED with
179 # async dependency solving and response validation, so exhausting it
180 # degrades async routes too. This app has ~248 sync routes against 65
181 # async ones, each sync request holding a worker for its full duration
182 # (including a first-call SQLCipher open), and it runs with workers=1,
183 # so there is no second process to absorb the overflow. Measured
184 # symptom: /api/v1/health went 0.8ms -> 9.8s at 80 concurrent requests,
185 # past the 8s Docker healthcheck timeout — i.e. the container reports
186 # unhealthy under load rather than merely slow.
187 #
188 # Left at AnyIO's default unless an operator opts in, so this changes
189 # nothing by default: raising it is not free (context-switch overhead,
190 # and more threads contend for the same per-user QueuePool), and the
191 # right value depends on the deployment's core count and workload. The
192 # point is to make an invisible framework default into a deliberate,
193 # tunable decision — the migration hazard here is not the number, it is
194 # that nobody knows the number exists.
195 try:
196 from ..settings.env_registry import (
197 get_env_setting as _get_env_setting,
198 )
200 configured = _get_env_setting("web.threadpool_max_threads")
201 if configured: 201 ↛ 202line 201 didn't jump to line 202 because the condition on line 201 was never true
202 import anyio.to_thread
204 limiter = anyio.to_thread.current_default_thread_limiter()
205 previous = limiter.total_tokens
206 limiter.total_tokens = int(configured)
207 logger.info(
208 "AnyIO worker pool resized: {} -> {} threads "
209 "(LDR_WEB_THREADPOOL_MAX_THREADS)",
210 previous,
211 limiter.total_tokens,
212 )
214 warn_if_threadpool_exceeds_db_pool(limiter.total_tokens)
215 except Exception:
216 # Never let a tuning knob stop the server from starting.
217 logger.exception(
218 "Could not apply web.threadpool_max_threads; "
219 "continuing with the AnyIO default"
220 )
222 # Route stdlib loggers through loguru
223 for name in ("uvicorn", "uvicorn.access", "uvicorn.error"):
224 uv_logger = logging.getLogger(name)
225 uv_logger.setLevel(logging.WARNING)
226 if not any(isinstance(h, InterceptHandler) for h in uv_logger.handlers):
227 uv_logger.addHandler(InterceptHandler())
229 ap_logger = logging.getLogger("apscheduler")
230 ap_logger.setLevel(logging.WARNING)
231 if not any(isinstance(h, InterceptHandler) for h in ap_logger.handlers):
232 ap_logger.addHandler(InterceptHandler())
234 # Log data location
235 from ..config.paths import get_data_directory
236 from ..database.encrypted_db import db_manager
238 data_dir = get_data_directory()
239 logger.info("=" * 60)
240 logger.info("DATA STORAGE INFORMATION")
241 logger.info("=" * 60)
242 logger.info(f"Data directory: {data_dir}")
243 logger.info(
244 "Databases: Per-user encrypted databases in encrypted_databases/"
245 )
246 if db_manager.has_encryption: 246 ↛ 249line 246 didn't jump to line 249 because the condition on line 246 was always true
247 logger.info("SECURITY: Databases are encrypted with SQLCipher.")
248 else:
249 logger.warning(
250 "SECURITY NOTICE: SQLCipher is not available - databases are NOT encrypted."
251 )
252 logger.info("=" * 60)
254 # Surface a cipher misconfiguration that otherwise only shows up as
255 # affected users getting "Invalid username or password": a relaxed
256 # SQLCipher KDF (test mode) on a deployment that already holds real user
257 # databases. No-op on fresh installs and when the effective KDF is at the
258 # production floor. Wrapped so a check failure can never block server boot.
259 try:
260 from ..database.sqlcipher_utils import (
261 warn_if_weak_kdf_with_existing_databases,
262 )
264 if db_manager.has_encryption: 264 ↛ 270line 264 didn't jump to line 270 because the condition on line 264 was always true
265 warn_if_weak_kdf_with_existing_databases(db_manager.data_dir)
266 except Exception:
267 logger.exception("Weak-KDF startup configuration check failed")
269 # Initialize Theme helper
270 from .themes import theme_registry
272 try:
273 static_dir = Path(STATIC_DIR)
274 themes_css_path = static_dir / "css" / "themes.css"
275 combined_css = theme_registry.get_combined_css()
276 themes_css_path.write_text(combined_css, encoding="utf-8")
277 logger.debug(
278 f"Generated themes.css with {len(theme_registry.themes)} themes"
279 )
280 except Exception:
281 logger.warning("Error generating combined themes.css")
283 # Start log queue processor (drains background-thread DB log entries).
284 #
285 # Wrapped like every other optional startup step in this lifespan (the
286 # AnyIO threadpool knob, the weak-KDF check, themes.css). The lifespan is
287 # two-tier and an UNGUARDED step that raises produces
288 # `lifespan.startup.failed`, so uvicorn never serves at all -- pinned by
289 # tests/web/test_lifespan_startup_shutdown.py.
290 # `start_log_queue_processor` ends in `threading.Thread(...).start()`,
291 # which raises `RuntimeError: can't start new thread` under a container
292 # pids / RLIMIT_NPROC ceiling; a best-effort logging daemon must never be
293 # the reason the server fails to boot. main guards the same call in
294 # `web/app.py::main()` (PR #3488) and the guard was lost in the port.
295 # The matching `stop_log_queue_processor()` on the shutdown path is
296 # itself wrapped and is a no-op when no daemon ever started, so it stays
297 # reachable either way; logging degrades to the shutdown flush.
298 from ..utilities.log_utils import start_log_queue_processor
300 try:
301 start_log_queue_processor()
302 except Exception:
303 logger.exception(
304 "Failed to start log queue processor; continuing without the "
305 "background log drain (queued entries flush at shutdown instead)"
306 )
308 # Start research queue processor — enabled by default in normal app runs,
309 # but default-off under pytest so app fixtures don't each spawn a daemon
310 # thread. An explicit web.queue_processor.enabled env override still wins
311 # (tests that need the processor opt in). Ports #5055 from the retired
312 # Flask app_factory to the FastAPI lifespan. The singleton is imported
313 # unconditionally (cheap — only .start() spawns the thread) so the
314 # shutdown path below can always call .stop().
315 from ..settings.env_registry import get_env_setting, registry
316 from .queue.processor_v2 import queue_processor
318 queue_processor_enabled = get_env_setting(
319 "web.queue_processor.enabled", True
320 )
321 _qp_setting = registry.get_setting_object("web.queue_processor.enabled")
322 _qp_env_set = bool(_qp_setting and _qp_setting.is_set)
323 if os.getenv("PYTEST_CURRENT_TEST") and not _qp_env_set: 323 ↛ 326line 323 didn't jump to line 326 because the condition on line 323 was always true
324 queue_processor_enabled = False
326 if queue_processor_enabled: 326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true
327 queue_processor.start()
328 logger.info("Started research queue processor v2")
329 else:
330 # Disabling the processor also disables the ONLY drain path for
331 # ``pending_operations``: research worker threads that cannot reach
332 # the DB directly (e.g. password lookup failed) fall back to
333 # ``queue_progress_update`` / ``queue_error_update``, and the only
334 # consumer of that queue is ``_drain_pending_operations``, which runs
335 # inside the loop started just above. Under Flask a second, always-on
336 # ``before_request`` hook drained it too; the FastAPI port has no
337 # equivalent. Directly-dispatched research still runs while the
338 # processor is off, so a terminal FAILED status raised on that
339 # fallback path would be queued and then silently dropped by TTL
340 # eviction. Warn rather than start a thread the operator turned off.
341 logger.warning(
342 "Queue processor v2 disabled — not starting. Queued researches "
343 "will not be dispatched, and progress/terminal-status updates "
344 "that fall back to the pending-operations queue will not be "
345 "persisted (they expire via TTL)."
346 )
348 # Start news scheduler
349 news_scheduler = None
350 try:
351 from ..settings.env_registry import get_env_setting
353 scheduler_enabled = get_env_setting("news.scheduler.enabled", True)
354 if scheduler_enabled: 354 ↛ 355line 354 didn't jump to line 355 because the condition on line 354 was never true
355 from ..scheduler.background import get_background_job_scheduler
356 from ..settings.manager import SettingsManager
358 # Startup-only: lifespan runs once before the server accepts
359 # traffic, so blocking here cannot stall in-flight requests.
360 settings_manager = SettingsManager() # allow: sync-in-async
361 news_scheduler = get_background_job_scheduler()
362 news_scheduler.initialize_with_settings(settings_manager)
363 news_scheduler.start()
364 logger.info("News scheduler started")
365 except Exception:
366 logger.exception("Failed to initialize news scheduler")
368 # Start connection cleanup scheduler
369 cleanup_scheduler = None
370 try:
371 from .auth.connection_cleanup import start_connection_cleanup_scheduler
372 from .auth.session_manager import session_manager
374 cleanup_scheduler = start_connection_cleanup_scheduler(
375 session_manager, db_manager
376 )
377 except Exception:
378 logger.warning("Failed to start cleanup scheduler")
380 # Shutdown is the code after `yield` below. A parallel atexit handler
381 # would double-close the DB on normal exit (lifespan runs first, then
382 # atexit fires on interpreter teardown), which is why there isn't one.
383 #
384 # CORRECTION (pre-merge readiness audit): this comment previously said
385 # "the lifespan `finally` below owns shutdown". There is no `finally` —
386 # the body is flat, `yield` then shutdown statements. That matters
387 # because the sentence was doing real work: it justified having no
388 # atexit handler by appealing to a guarantee the code does not make.
389 #
390 # What actually holds. uvicorn's graceful path stops accepting, drains
391 # connections, then drives the ASGI lifespan shutdown, which resumes
392 # this generator through a normal `__aexit__` — so on SIGTERM the code
393 # below DOES run. What is NOT covered is cancellation: if
394 # `timeout_graceful_shutdown` (10s, see web/app.py) expires or the
395 # process is killed harder, CancelledError is thrown in at the `yield`
396 # and everything below is skipped. Wrapping this in try/finally would
397 # close that, and is deliberately NOT done in the migration PR: it
398 # reindents ~48 lines on the shutdown path, and running full DB cleanup
399 # while already past the force-kill deadline is a judgement call for
400 # whoever owns deployment, not a mechanical fix.
401 #
402 # Consequence to know when reasoning about the atexit decision above:
403 # on a forced kill neither runs.
405 yield # --- App is running ---
407 # --- Shutdown ---
408 logger.info("Shutting down Local Deep Research...")
410 if news_scheduler: 410 ↛ 411line 410 didn't jump to line 411 because the condition on line 410 was never true
411 try:
412 news_scheduler.stop()
413 logger.info("News scheduler stopped")
414 except Exception:
415 logger.exception("Error stopping news scheduler")
417 if cleanup_scheduler: 417 ↛ 427line 417 didn't jump to line 427 because the condition on line 417 was always true
418 try:
419 # wait=True so in-flight cleanup jobs finish before we close
420 # the DB below — otherwise they hit a closed pool.
421 cleanup_scheduler.shutdown(wait=True)
422 except Exception:
423 logger.debug("Error stopping cleanup scheduler")
425 # Stop the research queue processor so a uvicorn reload doesn't leave
426 # an old daemon thread holding locks while the new process starts.
427 try:
428 queue_processor.stop()
429 except Exception:
430 logger.debug("Error stopping queue processor")
432 # Flush pending DB log writes BEFORE closing databases; `database_sink`
433 # swallows errors silently, so anything flushed after close is just
434 # dropped.
435 from ..utilities.log_utils import (
436 flush_log_queue,
437 stop_log_queue_processor,
438 )
440 try:
441 flush_log_queue()
442 stop_log_queue_processor()
443 except Exception:
444 logger.debug("Error flushing log queue during shutdown")
446 try:
447 db_manager.close_all_databases()
448 logger.info("Database connections closed")
449 except Exception:
450 logger.exception("Error closing databases")
453# ---------------------------------------------------------------------------
454# ASGI Middleware (pure ASGI — NOT BaseHTTPMiddleware)
455# ---------------------------------------------------------------------------
458def _is_private_ip(ip_str: str) -> bool:
459 """Check if IP is a private/local network address.
461 Returns True for private IPs (RFC 1918), localhost, and non-parseable
462 strings (e.g. 'testclient' from TestClient). Non-parseable strings
463 are treated as private to avoid adding Secure flag in test/dev contexts.
464 """
465 if not ip_str:
466 return True
467 try:
468 ip = ipaddress.ip_address(ip_str)
469 return ip.is_private or ip.is_loopback
470 except ValueError:
471 return True # Non-IP strings (e.g. "testclient") treated as private
474class SecurityHeadersMiddleware:
475 """Pure ASGI middleware for security headers.
477 Ports Flask's SecurityHeaders + ServerHeaderMiddleware to ASGI.
478 """
480 # Pre-compute static header values
481 CSP = (
482 "default-src 'self'; "
483 "connect-src 'self'; "
484 "script-src 'self' 'unsafe-inline'; "
485 "style-src 'self' 'unsafe-inline'; "
486 "font-src 'self' data:; "
487 "img-src 'self' data:; "
488 "media-src 'self'; "
489 "worker-src blob:; "
490 "child-src 'self' blob:; "
491 "frame-src 'self'; "
492 "frame-ancestors 'self'; "
493 "manifest-src 'self'; "
494 "object-src 'none'; "
495 "base-uri 'self'; "
496 "form-action 'self';"
497 )
498 PERMISSIONS = (
499 "geolocation=(), midi=(), camera=(), usb=(), "
500 "magnetometer=(), accelerometer=(), gyroscope=(), "
501 "microphone=(), payment=(), sync-xhr=(), document-domain=()"
502 )
504 @classmethod
505 def unconditional_headers(cls) -> list[tuple[bytes, bytes]]:
506 """The security headers stamped on every HTTP response.
508 Exposed separately because this middleware cannot cover every
509 response. Starlette's ``ServerErrorMiddleware`` sits OUTSIDE every
510 middleware added via ``app.add_middleware`` — it is installed by
511 ``build_middleware_stack`` itself — and when an exception handler
512 is registered for the bare ``Exception`` class, Starlette wires it
513 in as that middleware's handler. Its response is written to the raw
514 ASGI ``send``, bypassing this middleware entirely, so a 500 from an
515 unregistered exception type would otherwise carry NO security
516 headers at all. The catch-all handler reuses this list to stamp
517 them itself; keeping one source stops the two copies drifting.
519 Excludes the two conditional headers (HSTS, which needs a secure
520 scheme, and cache-control, which is skipped for /static/).
521 """
522 return [
523 (b"content-security-policy", cls.CSP.encode()),
524 (b"x-frame-options", b"SAMEORIGIN"),
525 (b"x-content-type-options", b"nosniff"),
526 (b"cross-origin-opener-policy", b"same-origin"),
527 (b"cross-origin-embedder-policy", b"credentialless"),
528 (b"cross-origin-resource-policy", b"same-origin"),
529 (b"permissions-policy", cls.PERMISSIONS.encode()),
530 (b"referrer-policy", b"strict-origin-when-cross-origin"),
531 ]
533 @classmethod
534 def cache_headers(cls) -> list[tuple[bytes, bytes]]:
535 """No-store cache headers stamped on every non-``/static/`` response.
537 Split out from ``unconditional_headers()`` (Cache-Control/Pragma/
538 Expires are conditional on path, not truly unconditional) so the
539 catch-all 500 handler below can reuse the exact same values
540 instead of hand-copying them — main's Flask ``after_request``
541 (``security/security_headers.py``) set all three:
542 ``Cache-Control``, ``Pragma`` (HTTP/1.0 compatibility), and
543 ``Expires: 0``. The ASGI port originally dropped ``Expires``;
544 restored here for parity. Impact of the omission was low
545 (``Cache-Control: no-store`` already dominates for any HTTP/1.1
546 cache) but it's a free one-line fix once the two are split out.
547 """
548 return [
549 (
550 b"cache-control",
551 b"no-store, no-cache, must-revalidate, max-age=0",
552 ),
553 (b"pragma", b"no-cache"),
554 (b"expires", b"0"),
555 ]
557 def __init__(self, app):
558 self.app = app
560 async def __call__(self, scope, receive, send):
561 if scope["type"] != "http":
562 await self.app(scope, receive, send)
563 return
565 path = scope.get("path", "")
566 is_secure = scope.get("scheme") == "https"
568 async def send_wrapper(message):
569 if message["type"] == "http.response.start":
570 headers = list(message.get("headers", []))
572 # Security headers
573 headers.extend(self.unconditional_headers())
575 # HSTS for secure connections
576 if is_secure:
577 headers.append(
578 (
579 b"strict-transport-security",
580 b"max-age=31536000; includeSubDomains",
581 )
582 )
584 # Cache control for non-static routes
585 if not path.startswith("/static/"):
586 headers.extend(self.cache_headers())
588 # Remove server header
589 headers = [(k, v) for k, v in headers if k.lower() != b"server"]
591 message = {**message, "headers": headers}
593 await send(message)
595 await self.app(scope, receive, send_wrapper)
598class _RequestBodyTooLarge(Exception):
599 """Raised by BodySizeLimitMiddleware's receive wrapper on overflow."""
602#: Separate, much smaller cap applied to non-multipart bodies (JSON and
603#: everything else that isn't a file upload) by ``BodySizeLimitMiddleware``.
604#:
605#: ``await request.json()`` -> ``json.loads`` is synchronous and runs on
606#: the single uvicorn event loop this app is served with (workers=1), so
607#: one caller's oversized body stalls every concurrent user. Measured on
608#: this branch: a 104 MB JSON body stalled the loop for 637 ms, during
609#: which 12 concurrent GETs spiked from ~3 ms to 184 ms latency. The only
610#: prior bound was the general ``max_body_size`` cap below
611#: (MAX_FILES_PER_REQUEST(200) x MAX_FILE_SIZE(3GB) ~= 600 GB) — sized for
612#: multipart file uploads, not a practical limit for a single JSON body.
613#:
614#: 100 MB matches the existing precedent in
615#: ``web/routers/notes.py::_MAX_JSON_BODY_BYTES``
616#: (``2 * NOTE_CONTENT_MAX_BYTES``, i.e. 2x the 50 MB note-content cap),
617#: which already accepts this order-of-magnitude stall for its own
618#: pre-parse JSON body gate. No other JSON endpoint in the app needs
619#: anywhere near this: research/chat/benchmark/settings JSON bodies are
620#: small query/config/message payloads (KB, not MB) — see PR review notes
621#: for the survey of ``await request.json()`` call sites. The two routes
622#: that genuinely need multi-hundred-MB bodies (``/api/upload/pdf`` and
623#: ``/library/api/collections/{id}/upload``) are multipart file uploads,
624#: not JSON, and keep the large ``max_body_size`` cap below.
625_DEFAULT_MAX_JSON_BODY_SIZE = 100 * 1024 * 1024 # 100 MB
627#: The cap every OTHER JSON route gets. 100 MB above is what the notes
628#: route needs, but it is a weak mitigation for the defect this cap exists
629#: to address: ``json.loads`` runs synchronously on the single uvicorn
630#: event loop (workers=1), so body size translates directly into a stall
631#: that freezes EVERY concurrent user, not just the caller. Measured on
632#: this branch: 104 MB -> 498-637 ms, 34 MB -> ~128 ms, 8 MB -> ~38 ms.
633#: A 100 MB cap therefore still permits a ~600 ms full-service stall,
634#: repeatable at will by any authenticated caller.
635#:
636#: 16 MB holds the worst case to roughly 60 ms while leaving ~4x headroom
637#: over the largest realistic legitimate body found in a survey of every
638#: ``await request.json()`` call site (~4 MB). Routes that genuinely need
639#: more are listed in _LARGE_JSON_BODY_PREFIXES and keep the 100 MB cap.
640_DEFAULT_MAX_SMALL_JSON_BODY_SIZE = 16 * 1024 * 1024 # 16 MB
642#: Path prefixes allowed the full ``_DEFAULT_MAX_JSON_BODY_SIZE``. Keep
643#: this list as short as possible: every entry is a route that can stall
644#: the event loop for hundreds of ms. ``notes`` earns it because
645#: NOTE_CONTENT_MAX_BYTES is 50 MB and its handler already does its own
646#: bounded pre-parse read (notes.py::_notes_json_body).
647_LARGE_JSON_BODY_PREFIXES = ("/notes/",)
649#: The only two routes that genuinely consume a multipart upload body
650#: (both do ``await request.form()`` and pull ``UploadFile``s out of it).
651#: The large ``max_body_size`` cap is granted on PATH, never on the
652#: client's declared Content-Type. Starlette's ``Request.json()`` is
653#: ``json.loads(await self.body())`` -- it never inspects Content-Type --
654#: so a body labelled ``multipart/form-data`` is still parsed as JSON by
655#: any route that asks for JSON. Choosing the cap from the header alone
656#: therefore handed every one of the app's ``request.json()`` routes the
657#: ~600 GB upload cap (200 files x 3 GB) for the cost of one mislabelled
658#: header, while the route went on parsing the body as JSON.
659_MULTIPART_UPLOAD_PATHS = frozenset({"/api/upload/pdf"})
660_MULTIPART_UPLOAD_RE = re.compile(r"^/library/api/collections/[^/]+/upload$")
663def _is_multipart_upload_path(path: str) -> bool:
664 """True for the two routes allowed to carry an upload-sized body."""
665 return path in _MULTIPART_UPLOAD_PATHS or bool(
666 _MULTIPART_UPLOAD_RE.match(path)
667 )
670class BodySizeLimitMiddleware:
671 """Pure ASGI middleware enforcing a global request-body size cap.
673 Ports Flask's ``MAX_CONTENT_LENGTH`` (app_factory set it to
674 ``MAX_FILES_PER_REQUEST * MAX_FILE_SIZE``; Werkzeug rejected any
675 larger body with 413) plus the ``@app.errorhandler(413)`` content
676 negotiation. Without it every body-reading code path — including the
677 CSRF middleware, which buffers form bodies to extract the token —
678 would buffer arbitrarily large request bodies into memory.
680 The declared Content-Length is checked before the app runs; bodies
681 streamed without one (chunked transfer) are counted chunk by chunk so
682 the cap cannot be bypassed.
684 On top of that global cap, non-multipart requests (JSON bodies, but
685 also anything with a missing/spoofed Content-Type — a route can call
686 ``await request.json()`` regardless of what Content-Type the client
687 sent, so gating strictly on ``application/json`` would leave that as
688 a bypass) are additionally capped at the much smaller
689 ``max_json_body_size``. Multipart requests (the two real upload
690 routes) are exempt from that smaller cap and still enforce only
691 ``max_body_size`` — see ``_DEFAULT_MAX_JSON_BODY_SIZE`` above for the
692 justification and measurements.
693 """
695 def __init__(
696 self,
697 app,
698 max_body_size: int | None = None,
699 max_json_body_size: int | None = None,
700 max_large_json_body_size: int | None = None,
701 ):
702 self.app = app
703 if max_body_size is None:
704 from ..security.file_upload_validator import FileUploadValidator
706 max_body_size = (
707 FileUploadValidator.MAX_FILES_PER_REQUEST
708 * FileUploadValidator.MAX_FILE_SIZE
709 )
710 self.max_body_size = max_body_size
711 # `max_json_body_size` is the cap for ORDINARY JSON routes, i.e.
712 # almost everything. `max_large_json_body_size` applies only to the
713 # few prefixes in _LARGE_JSON_BODY_PREFIXES that legitimately carry
714 # a big body. Keeping the ordinary case as the injectable default
715 # means a caller passing a small cap gets it applied where it
716 # matters, rather than silently only affecting notes.
717 if max_json_body_size is None:
718 max_json_body_size = _DEFAULT_MAX_SMALL_JSON_BODY_SIZE
719 self.max_json_body_size = max_json_body_size
720 if max_large_json_body_size is None:
721 max_large_json_body_size = _DEFAULT_MAX_JSON_BODY_SIZE
722 self.max_large_json_body_size = max(
723 max_large_json_body_size, max_json_body_size
724 )
726 async def _send_413(self, scope, send):
727 path = scope.get("path", "")
728 # Same negotiation as main's 413 errorhandler (_is_api_path).
729 if "/api/" in path or path.endswith("/api"):
730 body = b'{"error": "Request too large"}'
731 content_type = b"application/json"
732 else:
733 body = b"Request too large"
734 content_type = b"text/plain; charset=utf-8"
735 await send(
736 {
737 "type": "http.response.start",
738 "status": 413,
739 "headers": [
740 (b"content-type", content_type),
741 (b"content-length", str(len(body)).encode()),
742 ],
743 }
744 )
745 await send({"type": "http.response.body", "body": body})
747 async def __call__(self, scope, receive, send):
748 if scope["type"] != "http":
749 await self.app(scope, receive, send)
750 return
752 # Multipart ON one of the two real upload paths keeps the large
753 # cap; everything else — JSON, a mislabelled multipart body, and
754 # anything with a missing/other Content-Type, since a route can
755 # call `request.json()` no matter what Content-Type was sent — is
756 # additionally bounded by
757 # the much smaller JSON cap. `min()` also means an explicit,
758 # smaller `max_body_size` passed to the constructor (as tests
759 # do) still wins, so this never widens an existing caller's cap.
760 content_type = b""
761 for name, value in scope.get("headers", []):
762 if name == b"content-type":
763 content_type = value
764 break
765 # Both conditions are required: a multipart *label* is not enough,
766 # because it is client-supplied and costs nothing to forge (see
767 # _MULTIPART_UPLOAD_PATHS). Only the real upload routes get the
768 # upload-sized cap; a mislabelled body on any other path falls
769 # through to the JSON cap below, exactly as an honestly-labelled
770 # one does.
771 is_multipart = content_type.lower().startswith(
772 b"multipart/"
773 ) and _is_multipart_upload_path(scope.get("path", ""))
774 if is_multipart:
775 effective_max = self.max_body_size
776 else:
777 # Routes that legitimately carry a large JSON body keep the
778 # larger cap. Only notes qualifies: NOTE_CONTENT_MAX_BYTES is
779 # 50 MB, and notes.py::_MAX_JSON_BODY_BYTES doubles that to
780 # allow for JSON escaping. Every other `request.json()` call
781 # site in the app carries a small query/config/message payload
782 # — the largest realistic one is library_delete's bulk
783 # `document_ids`, ~4 MB even at 100k UUIDs.
784 json_cap = (
785 self.max_large_json_body_size
786 if scope.get("path", "").startswith(_LARGE_JSON_BODY_PREFIXES)
787 else self.max_json_body_size
788 )
789 effective_max = min(self.max_body_size, json_cap)
791 # Fast path: client declared an over-cap Content-Length.
792 for name, value in scope.get("headers", []):
793 if name == b"content-length":
794 try:
795 declared = int(value)
796 except ValueError:
797 break
798 if declared > effective_max:
799 await self._send_413(scope, send)
800 return
801 break
803 received = 0
804 response_started = False
806 async def recv_limited():
807 nonlocal received
808 message = await receive()
809 if message["type"] == "http.request": 809 ↛ 813line 809 didn't jump to line 813 because the condition on line 809 was always true
810 received += len(message.get("body", b""))
811 if received > effective_max:
812 raise _RequestBodyTooLarge()
813 return message
815 async def send_tracking(message):
816 nonlocal response_started
817 if message["type"] == "http.response.start":
818 response_started = True
819 await send(message)
821 try:
822 await self.app(scope, recv_limited, send_tracking)
823 except _RequestBodyTooLarge:
824 if response_started:
825 # Too late for a 413 — let the server tear the
826 # connection down.
827 raise
828 logger.warning(
829 f"Rejected over-limit request body on {scope.get('path', '')} "
830 f"(> {effective_max} bytes)"
831 )
832 await self._send_413(scope, send)
835class SecureCookieMiddleware:
836 """Pure ASGI middleware to add Secure flag to cookies based on client IP.
838 Ports Flask's SecureCookieMiddleware to ASGI.
839 """
841 def __init__(self, app, testing: bool = False):
842 self.app = app
843 self.testing = testing
844 self._warned_insecure_public = False
845 self._warned_untrusted_forwarded_proto = False
847 def _maybe_warn_untrusted_forwarded_proto(
848 self, scope, remote_addr: str
849 ) -> None:
850 """One-shot warning for the reverse-proxy misconfiguration that is
851 otherwise completely silent.
853 ``_maybe_warn_insecure_public`` below only fires for a NON-private
854 client IP, so the most common broken upgrade never trips it: nginx (or
855 caddy/traefik) terminating TLS on 127.0.0.1 with TRUST_PROXY_HEADERS
856 unset. The peer address is loopback, so that warning takes the
857 private-IP branch and says nothing, while uvicorn — not being told to
858 honour forwarded headers — reports scheme "http". The session cookie
859 then silently loses ``Secure`` and HSTS is silently withheld, on a
860 deployment the operator believes is HTTPS.
862 The tell is a request that carries ``X-Forwarded-Proto: https`` while
863 the connection is not being treated as HTTPS. Cheap: this only runs on
864 the not-https path (a correctly configured deployment never reaches
865 it), and short-circuits on a bool before touching headers, so it scans
866 them at most once per process.
867 """
868 if self._warned_untrusted_forwarded_proto:
869 return
870 # Set BEFORE deciding whether to warn, not after. The diagnosis is a
871 # per-process property, so scanning once is enough either way. Setting
872 # it only on the warn path (the original shape) meant the common
873 # deployment — plain HTTP with no proxy, so the header is absent —
874 # never set it and re-scanned every header of every request for the
875 # life of the process, which is the opposite of what this method's
876 # docstring promised.
877 self._warned_untrusted_forwarded_proto = True
879 # Only meaningful from a proxy on the loopback/private network. This
880 # header is untrusted client input: any remote client could otherwise
881 # send `X-Forwarded-Proto: https` once and plant a log line telling
882 # the operator to set TRUST_PROXY_HEADERS=true — advice that, followed
883 # on a host NOT behind a proxy, would hand that same client control of
884 # the scheme and therefore of the Secure-cookie and HSTS decisions.
885 if not remote_addr or not _is_private_ip(remote_addr):
886 return
888 for name, value in scope.get("headers", []):
889 if name != b"x-forwarded-proto":
890 continue
891 # Left-most value is the original client protocol when a chain of
892 # proxies has appended to the header.
893 if value.split(b",")[0].strip().lower() == b"https":
894 logger.warning(
895 "Received X-Forwarded-Proto: https but this connection is "
896 "not being treated as HTTPS, so session cookies will NOT "
897 "get the Secure flag and HSTS will NOT be sent. If this "
898 "server sits behind a TLS-terminating reverse proxy, set "
899 "TRUST_PROXY_HEADERS=true so forwarded headers are "
900 "honoured."
901 )
902 break
904 def _maybe_warn_insecure_public(self, remote_addr: str) -> None:
905 """One-shot warning when serving HTTP to a non-private client IP —
906 a likely missing HTTPS proxy. Mirrors main's operator signal."""
907 if self._warned_insecure_public:
908 return
909 if remote_addr and not _is_private_ip(remote_addr):
910 self._warned_insecure_public = True
911 logger.warning(
912 "Serving HTTP to a public client ({}). Session/CSRF cookies "
913 "are NOT marked Secure (marking them Secure over HTTP would "
914 "make the browser drop them). Put LDR behind HTTPS, and if "
915 "using a TLS proxy set TRUST_PROXY_HEADERS=true.",
916 remote_addr,
917 )
919 async def __call__(self, scope, receive, send):
920 if scope["type"] != "http":
921 await self.app(scope, receive, send)
922 return
924 # Add the Secure flag ONLY when the connection is actually HTTPS.
925 # Matches Flask's SecureCookieMiddleware (#3849): setting Secure on a
926 # cookie served over plain HTTP makes the browser DROP the cookie
927 # (RFC6265bis), so a deployment reachable over HTTP from a public IP
928 # would hit an unbreakable login loop. A previous version here added
929 # Secure for "public IP over HTTP" too, reintroducing exactly that
930 # bug. For a deployment behind a TLS-terminating proxy, honor
931 # X-Forwarded-Proto by starting uvicorn with TRUST_PROXY_HEADERS=true
932 # so scope["scheme"] reflects https (see web/app.py). We still warn
933 # once when serving HTTP to a non-private client so a missing HTTPS
934 # proxy is visible.
935 client = scope.get("client")
936 remote_addr = client[0] if client else ""
937 is_https = scope.get("scheme") == "https"
938 if not is_https and not self.testing:
939 self._maybe_warn_insecure_public(remote_addr)
940 self._maybe_warn_untrusted_forwarded_proto(scope, remote_addr)
941 should_add_secure = not self.testing and is_https
943 if not should_add_secure:
944 await self.app(scope, receive, send)
945 return
947 async def send_wrapper(message):
948 if message["type"] == "http.response.start":
949 headers = []
950 for k, v in message.get("headers", []):
951 if k.lower() == b"set-cookie":
952 v_str = v.decode("latin-1")
953 if "; Secure" not in v_str and "; secure" not in v_str:
954 v_str += "; Secure"
955 v = v_str.encode("latin-1")
956 headers.append((k, v))
957 message = {**message, "headers": headers}
958 await send(message)
960 await self.app(scope, receive, send_wrapper)
963class RememberMeMiddleware:
964 """Strip session-cookie Max-Age/Expires unless `_remember_me` is True.
966 Starlette's SessionMiddleware accepts a single `max_age` applied to
967 every session cookie. Without this middleware, every login gets a
968 30-day persistent cookie regardless of whether the user ticked
969 "Remember me" — which is the usual UX complaint, and would be a
970 privacy regression vs. the Flask app that honored the flag (Flask
971 defaulted `session.permanent = False`, i.e. a browser-session
972 cookie, and only made it persistent when "remember me" was
973 checked).
975 The login handler stores `_remember_me` inside the session dict.
976 On every response we inspect it and, unless it is explicitly
977 `True`, strip `Max-Age` and `Expires` attributes from the outbound
978 `session=...` Set-Cookie so the browser treats it as a session
979 cookie (discarded on browser close). Anonymous visitors (no login
980 yet) and requests right after logout also read back as "not True"
981 here — `_remember_me` is either absent (never logged in) or wiped
982 by `request.session.clear()` at logout (routers/auth.py), so it
983 reads back as `None`, not `False`. Both must get the same
984 non-persistent cookie as an explicit `_remember_me=False` login;
985 checking `remember_me is False` instead of `is not True` would miss
986 both cases and hand every anonymous visitor and every just-logged-out
987 user a persistent 30-day cookie.
989 This ONLY relaxes the client-side (browser) cap. The itsdangerous
990 signature backing the cookie is still valid for the full
991 `security.session_remember_me_days` window (SessionMiddleware has a
992 single `max_age` shared by every session — see its construction
993 below) regardless of this flag, so a raw non-remember-me cookie value
994 copied out of the browser (XSS exfiltration, proxy log, devtools)
995 would replay successfully for up to 30 days even though the browser
996 itself was told to discard it on close. `_enforce_session_expiry` /
997 `_stamp_session_expiry` (used by `DatabaseMiddleware` below) close
998 that gap by enforcing the shorter `security.session_timeout_hours`
999 window server-side, independent of the browser-facing cookie
1000 attributes this middleware controls.
1002 EXCEPTION — the deletion cookie must survive untouched. When a session
1003 is cleared server-side (idle-timeout revocation in
1004 `_enforce_session_revocation`, or logout), Starlette's SessionMiddleware
1005 does not write a normal data cookie: it writes the literal
1006 `session=null; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT; ...`
1007 (see `starlette.middleware.sessions.SessionMiddleware.send_wrapper`,
1008 the `session.modified and not initial_session_was_empty` branch) — an
1009 explicit "delete this cookie now" instruction, not persistent session
1010 data. Stripping `Expires`/`Max-Age` from THAT header, the same as any
1011 other non-remember-me cookie, downgrades it to `session=null; path=/;
1012 httponly; samesite=strict`: no expiry in the past, so the browser does
1013 not delete the cookie at all — it just keeps sending the literal string
1014 "null" back as the session value for the rest of the browser session.
1015 A revoked/logged-out session would then never actually leave the
1016 client. `session=null;` is the exact, fixed marker Starlette emits for
1017 a cleared session (never a signed payload), so matching it precisely
1018 identifies the deletion cookie without needing to duplicate
1019 SessionMiddleware's own emptiness bookkeeping.
1020 """
1022 # Name of the cookie Starlette's SessionMiddleware sets.
1023 _SESSION_COOKIE_NAME = b"session"
1025 # The literal value Starlette's SessionMiddleware writes when a session
1026 # is cleared (see the class docstring's EXCEPTION section). Distinct by
1027 # construction from every real session cookie, which carries a base64
1028 # itsdangerous-signed payload, never the bare string "null".
1029 _DELETED_COOKIE_PREFIX = _SESSION_COOKIE_NAME + b"=null;"
1031 def __init__(self, app) -> None:
1032 self.app = app
1034 async def __call__(self, scope, receive, send):
1035 if scope["type"] != "http":
1036 await self.app(scope, receive, send)
1037 return
1039 async def send_wrapper(message):
1040 if message["type"] != "http.response.start":
1041 await send(message)
1042 return
1044 # Resolve remember_me preference from the session. Treat
1045 # anything other than an explicit True (missing/None for
1046 # anonymous or just-logged-out sessions, or False for an
1047 # explicit non-remember-me login) as "strip the persistent
1048 # cookie attributes" — only an explicit opt-in keeps them.
1049 session = scope.get("session", {}) or {}
1050 remember_me = session.get("_remember_me")
1051 if remember_me is not True:
1052 rewritten = []
1053 for k, v in message.get("headers", []):
1054 if (
1055 k.lower() == b"set-cookie"
1056 and v.lower().startswith(
1057 self._SESSION_COOKIE_NAME + b"="
1058 )
1059 and not v.lower().startswith(
1060 self._DELETED_COOKIE_PREFIX
1061 )
1062 ):
1063 v_str = v.decode("latin-1")
1064 # Drop Max-Age=... and Expires=... attributes.
1065 parts = [
1066 part
1067 for part in v_str.split(";")
1068 if not part.strip()
1069 .lower()
1070 .startswith(("max-age=", "expires="))
1071 ]
1072 v = ";".join(parts).encode("latin-1")
1073 rewritten.append((k, v))
1074 message = {**message, "headers": rewritten}
1075 await send(message)
1077 await self.app(scope, receive, send_wrapper)
1080# ---------------------------------------------------------------------------
1081# Exception handlers
1082# ---------------------------------------------------------------------------
1085def _is_api_request(request: Request) -> bool:
1086 """Check if this is an API/JSON request (vs browser HTML request)."""
1087 path = request.url.path
1088 # Match paths containing /api/ segment or ending with /api
1089 if "/api/" in path or path.endswith("/api"):
1090 return True
1091 accept = request.headers.get("accept", "")
1092 if "application/json" in accept and "text/html" not in accept:
1093 return True
1094 return False
1097def _register_exception_handlers(app: FastAPI) -> None:
1098 """Register custom exception handlers (ports Flask's error handlers)."""
1100 from fastapi.exceptions import HTTPException
1102 from .exceptions import WebAPIException
1104 @app.exception_handler(HTTPException)
1105 async def handle_http_exception(request: Request, exc: HTTPException):
1106 """Redirect 401s for browser requests to login page (Flask parity)."""
1107 if exc.status_code == 401 and not _is_api_request(request):
1108 login_url = "/auth/login"
1109 # Path AND query. main used Werkzeug's `request.url` (the full
1110 # URL) as `next`, so a deep link survived the login bounce; the
1111 # port truncated it to the path, silently dropping the query.
1112 # That is not cosmetic: /chat/?q=... auto-submits the query as a
1113 # research run (static/js/components/chat.js routed-query bootstrap), so a
1114 # signed-out user following a shared link landed on an empty
1115 # chat box with their question gone. The consumer side already
1116 # handles this -- URLValidator.get_safe_redirect_path()
1117 # explicitly re-appends query and fragment, and is byte-identical
1118 # to main. quote(safe="/") below encodes the `?` and `&` so the
1119 # embedded value stays a single well-formed query parameter.
1120 next_url = str(request.url.path)
1121 if request.url.query:
1122 next_url += "?" + request.url.query
1123 if next_url and next_url != "/":
1124 # URL-encode the path before embedding as a query value.
1125 # Without quote(), characters like `?` or `&` in the path
1126 # break the resulting URL; downstream code that treats
1127 # ?next= as untrusted input must still validate it's a
1128 # local path, but at minimum the encoding here keeps the
1129 # redirect target well-formed.
1130 from urllib.parse import quote
1132 login_url += f"?next={quote(next_url, safe='/')}"
1133 return RedirectResponse(
1134 url=login_url,
1135 status_code=302,
1136 headers=exc.headers,
1137 )
1138 path = request.url.path
1139 if path == "/api/v1" or path.startswith("/api/v1/"):
1140 # /api/v1 is the documented programmatic API for external clients,
1141 # and main returned {"error": ...} for its auth failures
1142 # (web/api.py's `jsonify({"error": "Authentication required"})` /
1143 # `{"error": "API access is disabled"}`). Routing those through
1144 # HTTPException here would silently change the envelope to
1145 # {"detail": ...} for every existing script.
1146 #
1147 # Both keys are emitted rather than swapping: the frontend's
1148 # api.js reads `detail`, and tests/web/test_exception_handler_
1149 # contract.py deliberately pins the {"detail": ...} shape for the
1150 # rest of the app. This restores main's contract for the external
1151 # API without disturbing either.
1152 return JSONResponse(
1153 {"error": exc.detail, "detail": exc.detail},
1154 status_code=exc.status_code,
1155 headers=exc.headers,
1156 )
1157 return JSONResponse(
1158 {"detail": exc.detail},
1159 status_code=exc.status_code,
1160 headers=exc.headers,
1161 )
1163 @app.exception_handler(404)
1164 async def not_found(request: Request, exc):
1165 # Browser navigations get HTML; only API callers get JSON. Without
1166 # the branch, a user who mistypes a URL is shown a raw
1167 # ``{"error": "Not found"}`` body in the browser's JSON viewer.
1168 # Flask branched the same way — app_factory.py's
1169 # ``@app.errorhandler(404)`` returned ``make_response("Not found",
1170 # 404)``, which Flask serves as text/html — and the 401 handler
1171 # directly above already makes this distinction. This one did not.
1172 if _is_api_request(request):
1173 return JSONResponse({"error": "Not found"}, status_code=404)
1174 return HTMLResponse("Not found", status_code=404)
1176 # Catch-all so unhandled exceptions get logged with traceback rather than
1177 # silently disappearing into Starlette's default 500 path. (The Exception
1178 # handler takes precedence over status-code 500 via MRO, so a separate
1179 # 500 handler would be unreachable.)
1180 @app.exception_handler(Exception)
1181 async def unhandled_exception(request: Request, exc):
1182 from fastapi import HTTPException
1184 # Let HTTPException pass through to FastAPI's default handler.
1185 if isinstance(exc, HTTPException): 1185 ↛ 1186line 1185 didn't jump to line 1186 because the condition on line 1185 was never true
1186 raise exc from None
1187 logger.opt(exception=exc).error(
1188 "Unhandled exception: {} {}", request.method, request.url.path
1189 )
1190 # Stamp the security headers here rather than relying on
1191 # SecurityHeadersMiddleware. Registering a handler for the bare
1192 # ``Exception`` class makes Starlette wire it into
1193 # ``ServerErrorMiddleware``, which is installed OUTSIDE every
1194 # ``add_middleware`` layer and writes to the raw ASGI ``send`` —
1195 # so this response never passes through that middleware and would
1196 # otherwise ship with no CSP, no nosniff, no frame-options — and,
1197 # same reasoning, no Cache-Control/Pragma/Expires either (main's
1198 # Flask ``after_request`` covered every response including
1199 # unhandled 500s; this reuses SecurityHeadersMiddleware.
1200 # cache_headers() so the two can't drift apart).
1201 # NOTE: unlike the 404 handler above, this deliberately does NOT
1202 # branch on _is_api_request(). Flask's @app.errorhandler(500) did
1203 # (returning "Server error" as text/html for non-API paths), but the
1204 # JSON body here is a pinned contract — tests/web/
1205 # test_exception_handler_contract.py::Test500Contract and
1206 # test_middleware_order_and_headers.py both assert it — and a raw
1207 # JSON 500 is far less user-visible than a raw JSON 404, which users
1208 # hit routinely by mistyping a URL or following a stale link.
1209 # Changing it buys little and churns a contract two suites depend
1210 # on. Revisit alongside a real styled error page.
1211 return JSONResponse(
1212 {"error": "Server error"},
1213 status_code=500,
1214 headers={
1215 name.decode(): value.decode()
1216 for name, value in (
1217 SecurityHeadersMiddleware.unconditional_headers()
1218 + SecurityHeadersMiddleware.cache_headers()
1219 )
1220 },
1221 )
1223 # Flask parity: request.get_json() returned 400 on malformed JSON and on
1224 # request bodies that could not be decoded as UTF-8. Without these two
1225 # registrations a bare `await request.json()` surfaced either exception
1226 # as a 500 via the catch-all above.
1227 import json
1229 @app.exception_handler(UnicodeDecodeError)
1230 @app.exception_handler(json.JSONDecodeError)
1231 async def handle_json_decode_error(request: Request, exc):
1232 # Log it: this handler also catches a JSONDecodeError raised deeper
1233 # in a handler (e.g. parsing a downstream response), which is a
1234 # server-side fault wearing a 400. Without this line those would
1235 # vanish silently — the catch-all above logs everything it takes.
1236 #
1237 # Bounded fields only, never `exc` itself and never `exc.doc`.
1238 # JSONDecodeError carries the ENTIRE offending document on `.doc`,
1239 # so interpolating the exception is one __str__ change away from
1240 # putting a request body — or a downstream provider's response — in
1241 # the log. `.msg` is the parser's own text ("Expecting value"),
1242 # `.lineno`/`.colno` are ints. This also matches how every sibling
1243 # handler here logs (error_code / status_code / reason), and closes
1244 # a real blind spot in check-sensitive-logging, which tracks
1245 # exception variables via `except ... as e` and therefore cannot
1246 # see one arriving as an exception-handler parameter.
1247 logger.warning(
1248 "JSON decode error handling {} {}: {} (line {} column {})",
1249 request.method,
1250 request.url.path,
1251 getattr(exc, "msg", "invalid JSON"),
1252 getattr(exc, "lineno", "?"),
1253 getattr(exc, "colno", "?"),
1254 )
1255 return JSONResponse({"error": "Invalid JSON body"}, status_code=400)
1257 @app.exception_handler(WebAPIException)
1258 async def handle_web_api_exception(request: Request, exc: WebAPIException):
1259 logger.error(
1260 "Web API error: {} (status {})", exc.error_code, exc.status_code
1261 )
1262 return JSONResponse(exc.to_dict(), status_code=exc.status_code)
1264 try:
1265 from ..news.exceptions import NewsAPIException
1267 @app.exception_handler(NewsAPIException)
1268 async def handle_news_api_exception(
1269 request: Request, exc: NewsAPIException
1270 ):
1271 logger.error(
1272 "News API error: {} (status {})",
1273 exc.error_code,
1274 exc.status_code,
1275 )
1276 return JSONResponse(exc.to_dict(), status_code=exc.status_code)
1277 except ImportError:
1278 pass
1280 # PolicyDeniedError can escape any synchronous request-path PEP
1281 # (LLM, embeddings, URL fetch) that wasn't caught at the call site.
1282 # Without this handler it surfaces as a 500 — turn it into a clean
1283 # 400 with the decision reason so the user (and the operator
1284 # audit log) sees a policy-shaped error instead of a stack trace.
1285 try:
1286 from ..security.egress.policy import PolicyDeniedError
1288 @app.exception_handler(PolicyDeniedError)
1289 async def handle_policy_denied(
1290 request: Request, exc: PolicyDeniedError
1291 ):
1292 reason = getattr(getattr(exc, "decision", None), "reason", "denied")
1293 target = getattr(exc, "target", "")
1294 logger.bind(policy_audit=True).warning(
1295 "PolicyDeniedError surfaced at FastAPI exception handler",
1296 reason=reason,
1297 target=target,
1298 )
1299 return JSONResponse(
1300 {
1301 "status": "error",
1302 "message": (
1303 f"Egress policy refused this request: {reason}"
1304 ),
1305 },
1306 status_code=400,
1307 )
1308 except ImportError:
1309 # egress_policy is in-tree; skipping the handler is only useful
1310 # for partial test builds without the security package.
1311 pass
1314# ---------------------------------------------------------------------------
1315# Database middleware (before each request)
1316# ---------------------------------------------------------------------------
1318# Regression #1 fix — see _enforce_session_expiry's docstring for the full
1319# mechanism. Read once at import time, matching how SessionMiddleware's own
1320# `max_age` below is computed: an operator change to
1321# security.session_timeout_hours takes effect on the next restart, not live.
1322_NON_REMEMBER_ME_SESSION_SECONDS = (
1323 get_security_default("security.session_timeout_hours", 2) * 3600
1324)
1326_SESSION_EXPIRES_AT_KEY = "_session_expires_at"
1329def _now_ts() -> float:
1330 """Wall-clock now, as epoch seconds.
1332 A thin wrapper around ``time.time()`` purely so tests can monkeypatch
1333 session-expiry time (``local_deep_research.web.fastapi_app._now_ts``)
1334 without touching the process-global stdlib clock.
1335 """
1336 return time.time()
1339def _enforce_session_expiry(session) -> None:
1340 """Clear `session` in place if its server-side deadline has passed.
1342 Regression #1: Starlette's ``SessionMiddleware`` verifies the
1343 itsdangerous signature with a single ``max_age`` shared by every
1344 session (see its construction below) — the signer has no notion of
1345 "this particular session should expire sooner." That means a
1346 non-remember-me login's raw cookie value stays independently
1347 replayable for the FULL ``security.session_remember_me_days`` window
1348 (30 days) purely because that's what SessionMiddleware was built
1349 with, even though the browser is told (by ``RememberMeMiddleware``)
1350 to discard the cookie on close. The intended lifetime for a
1351 non-remember-me session is the much shorter
1352 ``security.session_timeout_hours`` (default 2h).
1354 Mechanism chosen: stamp an expiry deadline into the session payload
1355 itself (`_stamp_session_expiry` below) and enforce it here, in
1356 application code, independent of itsdangerous's own signature check.
1358 Alternatives considered and rejected:
1359 * Two ``SessionMiddleware`` instances (short/long ``max_age``),
1360 dispatched by ``_remember_me`` — the dispatch decision itself
1361 requires reading the session, i.e. doing the same "peek before you
1362 verify" work this function already does, for substantially more
1363 code (two secrets or cookie names to keep in sync, or a
1364 request-splitting wrapper ASGI app in front of both).
1365 * A custom itsdangerous signer with a variable ``max_age`` — ``max_age``
1366 is a verification-time argument to ``signer.unsign()``, not part of
1367 the signed payload, so "vary the signer" still needs some per-session
1368 signal to pick which ``max_age`` to verify with BEFORE verification
1369 succeeds. That signal is exactly the deadline stamped here — no
1370 simpler, and itsdangerous's own timestamp isn't readable without a
1371 successful unsign first (chicken-and-egg).
1372 * Enforce the same shorter deadline on remember-me sessions too
1373 (uniform handling, less code) — rejected: their itsdangerous-level
1374 cap already matches ``security.session_remember_me_days`` exactly,
1375 so adding a second, redundant deadline only risks the two drifting
1376 apart if an operator changes one setting and not the other.
1378 Called from ``DatabaseMiddleware.__call__`` — the innermost
1379 app-level middleware, run immediately before Starlette's router —
1380 specifically so this runs BEFORE CSRFMiddleware's own session read
1381 and before any route handler (including ``require_auth`` /
1382 ``/auth/check``) can act on a session whose signature is still valid
1383 but whose intended lifetime has passed. Clearing the dict here makes
1384 an expired session indistinguishable from "never logged in" to every
1385 downstream consumer, without touching any of those call sites.
1387 Existing sessions: does NOT force-log-out non-remember-me sessions
1388 that predate this fix. Their cookie has no stamped deadline yet
1389 (`_session_expires_at` absent is treated as "not yet due", not
1390 "expired"), so this is a no-op for them on their first request after
1391 deploy — `_stamp_session_expiry` gives them a fresh deadline on that
1392 same response instead. The one residual gap this leaves: a
1393 non-remember-me cookie captured (XSS/log/devtools) in the narrow
1394 window between deploy and that session's next request keeps its old,
1395 unbounded-until-30-days replay property for one more grant of
1396 ``security.session_timeout_hours``. A hard cutover (immediate mass
1397 logout of every non-remember-me session) would close that but was
1398 rejected per this task's explicit instruction not to log everyone out
1399 unnecessarily.
1400 """
1401 if session.get("_remember_me") is not False:
1402 return
1403 expires_at = session.get(_SESSION_EXPIRES_AT_KEY)
1404 if expires_at is not None and _now_ts() >= expires_at:
1405 session.clear()
1408def _stamp_session_expiry(session) -> None:
1409 """(Re)stamp the non-remember-me deadline `_enforce_session_expiry`
1410 checks, sliding it forward on every response for an authenticated,
1411 non-remember-me session. No-op for anonymous or remember-me sessions.
1413 Runs from `DatabaseMiddleware`'s response-phase `send_wrapper` — i.e.
1414 after any route handler (including the login handler itself) has
1415 already run — so the very first Set-Cookie a non-remember-me login
1416 produces already carries a deadline, not just its second response
1417 onward. Sliding (refreshed on every authenticated response, like
1418 ``session_manager.SessionManager``'s ``last_access``-based timeout)
1419 rather than a fixed deadline from login time, matching
1420 ``security.session_timeout_hours``'s documented meaning: "Session
1421 timeout ... for sessions without 'Remember Me' checked" — an idle
1422 timeout, not a hard cap on total session length.
1423 """
1424 if session.get("username") and session.get("_remember_me") is False:
1425 session[_SESSION_EXPIRES_AT_KEY] = (
1426 _now_ts() + _NON_REMEMBER_ME_SESSION_SECONDS
1427 )
1430def _enforce_session_revocation(session) -> None:
1431 """Clear `session` in place if its server-side record is gone.
1433 `require_auth` rejects a cookie whose `session_id` no longer resolves
1434 to the claimed username, which is what makes logout actually revoke a
1435 session rather than merely ask the browser to forget it. But two GET
1436 routes deliberately bypass `require_auth` — `/` and `/auth/check` —
1437 because they must render/answer for anonymous visitors too, and both
1438 read `request.session["username"]` directly. A sweep of all 110 GET
1439 routes found exactly those two granting a revoked session strictly
1440 more than an anonymous one: after logout, a replayed cookie still
1441 rendered the authenticated index and still got `authenticated: true`,
1442 for the full remaining itsdangerous window. `/` additionally opens
1443 `get_user_db_session(username)` without a `session_id`, so the page
1444 renders with the user's real saved settings.
1446 Enforcing it here rather than patching those two handlers means the
1447 property holds for every route by construction — including any future
1448 handler that reads the session without going through `require_auth`,
1449 which is precisely how these two got missed. Same placement rationale
1450 as `_enforce_session_expiry`: clearing the dict makes a revoked
1451 session indistinguishable from "never logged in" to everything
1452 downstream, so no call site needs to know about revocation.
1454 Anonymous sessions return immediately, so the cost on the hot path is
1455 one dict lookup; for authenticated ones it is `validate_session`'s
1456 in-memory dict read under a lock. That call also refreshes
1457 `last_access`, which is why this runs unconditionally rather than
1458 behind `_skip_prefixes` — same reasoning as the expiry check above.
1460 Note `session_manager.sessions` is in-memory, so a server restart
1461 invalidates every session. That is pre-existing `require_auth`
1462 behaviour, not something this widens: it already applied to every
1463 authenticated route. The only change is that `/` and `/auth/check`
1464 now agree with the rest of the app about who is logged in.
1465 """
1466 username = session.get("username")
1467 if not username:
1468 return
1470 from .auth.session_manager import session_manager
1472 session_id = session.get("session_id")
1473 if not session_id or session_manager.validate_session(session_id) != (
1474 username
1475 ):
1476 session.clear()
1479class DatabaseMiddleware:
1480 """Pure ASGI middleware to ensure user database is open for authenticated requests.
1482 Runs INSIDE SessionMiddleware so request.session is available. Also
1483 enforces/refreshes the non-remember-me session deadline (regression
1484 #1 — see `_enforce_session_expiry` / `_stamp_session_expiry` above)
1485 since this is the innermost app-level middleware: the last point to
1486 act on the session before CSRF's own read of it and before any route
1487 handler runs.
1488 """
1490 _skip_prefixes = (
1491 "/static/",
1492 "/favicon.ico",
1493 "/api/v1/health",
1494 "/auth/login",
1495 "/auth/register",
1496 "/auth/csrf-token",
1497 "/ws/",
1498 )
1500 def __init__(self, app):
1501 self.app = app
1503 async def __call__(self, scope, receive, send):
1504 if scope["type"] != "http":
1505 await self.app(scope, receive, send)
1506 return
1508 path = scope.get("path", "")
1509 method = scope.get("method", "GET")
1511 from ..utilities.request_context import (
1512 reset_request_user,
1513 set_request_user,
1514 )
1516 # Regression #1: enforce the shorter non-remember-me deadline
1517 # BEFORE anything downstream (CSRF's session read, the route
1518 # handler) can act on a session whose itsdangerous signature is
1519 # still valid but whose intended lifetime has passed. Runs
1520 # unconditionally (not gated by _skip_prefixes below) — it's a
1521 # cheap in-memory check/clear, not a DB open, and skipping it for
1522 # e.g. static-asset requests would let an idle-but-signature-valid
1523 # session's deadline silently stop advancing/enforcing whenever
1524 # the user's tab is only pulling static assets.
1525 session = scope.get("session")
1526 if session is not None:
1527 _enforce_session_expiry(session)
1528 # Same placement, same reason, different property: expiry is
1529 # "this session has been idle too long", revocation is "this
1530 # session was destroyed server-side (logout, password change)
1531 # but its signed cookie is still replayable". `require_auth`
1532 # covers the latter for the routes that use it; this covers
1533 # the ones that deliberately don't. See the docstring.
1534 _enforce_session_revocation(session)
1536 ctx_tokens = None
1537 # Run the inner app first to let SessionMiddleware populate scope
1538 # Then ensure database is open
1539 if method != "OPTIONS" and not any(
1540 path.startswith(p) for p in self._skip_prefixes
1541 ):
1542 # Session data is in scope["session"] after SessionMiddleware
1543 session_data = scope.get("session", {})
1544 if session_data:
1545 from .dependencies.auth import ensure_user_database
1546 from .dependencies.threadpool import run_db_sync
1548 # Create a minimal request-like object for ensure_user_database
1549 class _MinimalRequest:
1550 def __init__(self, session):
1551 self.session = session
1553 # ensure_user_database opens a SQLCipher connection (PBKDF2
1554 # key derivation, file I/O) which would block the event loop
1555 # for hundreds of ms on first call after login. Offload to a
1556 # threadpool so concurrent requests don't serialize behind it.
1557 #
1558 # Use run_db_sync, not bare asyncio.to_thread: a cold
1559 # open_user_database() call runs the phase-2 rekey path
1560 # (_run_phase2_rekey -> rekey_user_indexes), which opens a
1561 # get_user_db_session() unconditionally, before its own
1562 # marker check (vector_stores/legacy_rekey.py). That leaves a
1563 # thread-local session on whichever asyncio default-executor
1564 # thread served this call, with no cleanup boundary --
1565 # asyncio.to_thread's executor is not drained by
1566 # DatabaseMiddleware (event-loop thread only) or by any
1567 # per-worker cleanup, only ever replaced when that pooled
1568 # thread later serves a different username. run_db_sync
1569 # exists for exactly this: it runs the sync call on the same
1570 # default executor, then cleans up the thread-local DB
1571 # session (and other ambient thread-local state) on that
1572 # worker before returning.
1573 await run_db_sync(
1574 ensure_user_database, _MinimalRequest(session_data)
1575 )
1577 # Publish username/session_id to a contextvar so service-layer
1578 # code that historically read flask_session can fall back
1579 # without requiring a Flask request context.
1580 ctx_tokens = set_request_user(
1581 session_data.get("username"),
1582 session_data.get("session_id"),
1583 )
1585 async def send_wrapper(message):
1586 if message["type"] == "http.response.start":
1587 # Regression #1: (re)stamp the non-remember-me deadline on
1588 # the way out, after the route handler (including a fresh
1589 # login) has had a chance to set username/_remember_me.
1590 # See _stamp_session_expiry's docstring.
1591 current_session = scope.get("session")
1592 if current_session is not None:
1593 _stamp_session_expiry(current_session)
1594 await send(message)
1596 try:
1597 await self.app(scope, receive, send_wrapper)
1598 finally:
1599 if ctx_tokens is not None:
1600 reset_request_user(ctx_tokens)
1602 # Cleanup after request.
1603 #
1604 # Keep the two sweeps independent so one failure cannot skip the
1605 # other. This async middleware can clean event-loop-owned state;
1606 # WorkerCleanupAPIRoute already reclaimed a synchronous
1607 # endpoint's AnyIO-worker state before control returned here.
1608 # Pinned by tests/web/test_teardown_cleanup_asgi.py and
1609 # tests/web/test_db_session_lifecycle_asgi.py.
1610 try:
1611 from ..database.thread_local_session import (
1612 cleanup_dead_threads,
1613 )
1615 cleanup_dead_threads()
1616 except Exception:
1617 logger.debug("Error during post-request dead-thread cleanup")
1619 try:
1620 from ..database.thread_local_session import (
1621 cleanup_current_thread,
1622 )
1624 cleanup_current_thread()
1625 except Exception:
1626 logger.debug("Error during post-request current-thread cleanup")
1629# ---------------------------------------------------------------------------
1630# Request-arrival/duration forensics (CI/TESTING only) — issue #4431
1631# ---------------------------------------------------------------------------
1634class RequestTimingASGIMiddleware:
1635 """Pure ASGI port of ``web/utils/request_timing.RequestTimingMiddleware``.
1637 Named ``...ASGIMiddleware`` rather than reusing the original's name on
1638 purpose: both classes live in the same package, and one of them is a
1639 WSGI callable that would fail silently and confusingly if handed to
1640 ``add_middleware``.
1642 That module is a plain WSGI callable (``__call__(environ,
1643 start_response)``) — main installed it by wrapping ``app.wsgi_app`` in
1644 ``app_factory.create_app`` under CI/TESTING (#4536), and the FastAPI
1645 migration deleted ``app_factory.py`` and lost the wiring with it (#5959).
1646 A WSGI callable cannot be handed to ``app.add_middleware``, so the
1647 transport half is re-expressed here while the *policy* half — the log
1648 format the CI log-grep keys on, the slow-completion threshold, and the
1649 faulthandler freeze dead-man's switch — is imported from the original
1650 module so there is exactly one definition of each.
1652 Pure ASGI rather than ``BaseHTTPMiddleware`` for the same reason as
1653 ``BodySizeLimitMiddleware`` / ``_PathScopedCORSMiddleware``: this sits
1654 outside the streaming SSE routes and ``BaseHTTPMiddleware`` buffers
1655 ``StreamingResponse`` bodies.
1657 One deliberate difference from the WSGI original: the duration here
1658 covers the whole downstream ASGI call, i.e. arrival through the final
1659 response body — which is what #4431 actually wants to measure — rather
1660 than WSGI view execution with response streaming excluded. The arrival
1661 line, whose *absence* during a navigation timeout is the real signal,
1662 is byte-identical.
1663 """
1665 def __init__(self, app):
1666 self.app = app
1667 from .utils.request_timing import _arm_freeze_dump
1669 # Arm the freeze thread-dump dead-man's switch (no-op under pytest).
1670 _arm_freeze_dump()
1672 async def __call__(self, scope, receive, send):
1673 if scope["type"] != "http":
1674 # CORS-style scoping: the forensics are about HTTP request
1675 # arrival. lifespan/websocket scopes pass straight through.
1676 await self.app(scope, receive, send)
1677 return
1679 from .utils.request_timing import (
1680 SLOW_REQUEST_SECONDS,
1681 _arm_freeze_dump,
1682 )
1684 # Re-arm the dead-man's switch: as long as requests keep arriving the
1685 # dump never fires; a freeze (no arrivals) lets it fire and capture
1686 # the stuck thread stacks.
1687 _arm_freeze_dump()
1689 method = scope.get("method", "-")
1690 path = scope.get("path", "-")
1691 # engine.io transport/sid make poll churn correlatable. (sid is
1692 # logged on purpose for correlation; logs are CI-only artifacts.)
1693 if path.startswith("/socket.io"):
1694 query = scope.get("query_string", b"").decode("latin-1", "replace")
1695 path = f"{path}?{query}" if query else path
1696 # Strip CR/LF so a crafted path/query cannot inject fake log lines
1697 # (the forensics output is grep'd downstream).
1698 path = path.replace("\r", "\\r").replace("\n", "\\n")
1699 logger.info(f"[req] > {method} {path}")
1700 start = time.monotonic()
1701 try:
1702 await self.app(scope, receive, send)
1703 finally:
1704 elapsed = time.monotonic() - start
1705 if elapsed >= SLOW_REQUEST_SECONDS:
1706 logger.warning(f"[req] < {method} {path} {elapsed:.1f}s SLOW")
1707 else:
1708 logger.info(f"[req] < {method} {path} {elapsed:.2f}s")
1711def _request_timing_enabled() -> bool:
1712 """Whether to install :class:`RequestTimingASGIMiddleware` on this process.
1714 main's gate was ``os.environ.get("CI") or os.environ.get("TESTING")``,
1715 kept verbatim, plus one addition: never inside a pytest process.
1717 The diagnostic exists for the long-running UI-shard server, which CI
1718 starts as its own process (``python -m local_deep_research.web.app``)
1719 with ``CI`` set — pytest is not imported there, so the gate still arms
1720 exactly where #4431 needs it. The pytest exclusion mirrors
1721 ``request_timing._should_arm_freeze_dump``'s own reasoning (that module
1722 already refuses to arm the faulthandler timer under pytest) and keeps
1723 the in-process middleware stack that
1724 ``tests/web/test_middleware_order_and_headers.py`` pins byte-identical
1725 on a CI runner, where ``CI=true`` is set for the unit-test job too.
1726 """
1727 if "pytest" in sys.modules:
1728 return False
1729 return bool(os.environ.get("CI") or os.environ.get("TESTING"))
1732# ---------------------------------------------------------------------------
1733# Rate limiting
1734# ---------------------------------------------------------------------------
1737def _setup_rate_limiting(app: FastAPI) -> None:
1738 """Configure slowapi rate limiting on the app."""
1739 from slowapi.errors import RateLimitExceeded
1740 from slowapi.middleware import SlowAPIMiddleware
1742 from .dependencies.rate_limit import _get_client_ip, limiter
1744 app.state.limiter = limiter
1746 # Deliberately `def`, not `async def`. slowapi's SlowAPIMiddleware handles
1747 # the global default limits through `sync_check_limits`, which cannot await
1748 # and therefore DISCARDS an async handler outright:
1749 #
1750 # # cannot execute asynchronous code in a synchronous middleware,
1751 # # -> fallback on default exception handler
1752 # if inspect.iscoroutinefunction(exception_handler):
1753 # exception_handler = _rate_limit_exceeded_handler
1754 # (slowapi/middleware.py)
1755 #
1756 # While this was `async def`, every 429 raised by the middleware path — i.e.
1757 # every undecorated route hitting the global default — silently used
1758 # slowapi's stock handler instead of this one, losing the audit warning
1759 # below, the Retry-After headers, and main's {"error", "message"} body
1760 # contract. Only routes carrying an explicit @limiter.limit decorator
1761 # (the async path) ever reached this function. Nothing here awaits, so
1762 # making it sync costs nothing and covers both paths.
1763 def _rate_limit_exceeded(request: Request, exc: RateLimitExceeded):
1764 # Audit logging for security monitoring — port of main's 429
1765 # errorhandler. _get_client_ip resolves the real IP behind
1766 # trusted proxies (raw request.client would log the proxy).
1767 logger.warning(
1768 f"Rate limit exceeded: endpoint={request.url.path} "
1769 f"ip={_get_client_ip(request)} "
1770 f"user_agent={request.headers.get('User-Agent', 'unknown')}"
1771 )
1772 response = JSONResponse(
1773 {
1774 "error": "Too many requests",
1775 "message": "Too many attempts. Please try again later.",
1776 },
1777 status_code=429,
1778 # Stamp the security headers here, the same way the catch-all 500
1779 # handler does. `_setup_rate_limiting` runs AFTER
1780 # `add_middleware(SecurityHeadersMiddleware)`, and Starlette's
1781 # add_middleware is LIFO, so SlowAPIMiddleware ends up OUTSIDE
1782 # SecurityHeadersMiddleware — a 429 it generates never passes
1783 # through it and would otherwise ship with no CSP, no nosniff, no
1784 # X-Frame-Options and no Cache-Control. Reordering the stack would
1785 # be the alternative, but the order is deliberately pinned by
1786 # tests/web/test_middleware_order_and_headers.py.
1787 headers={
1788 name.decode(): value.decode()
1789 for name, value in (
1790 SecurityHeadersMiddleware.unconditional_headers()
1791 + SecurityHeadersMiddleware.cache_headers()
1792 )
1793 },
1794 )
1795 # Restore Retry-After / X-RateLimit-* on 429s. `headers_enabled=True`
1796 # on the Limiter (dependencies/rate_limit.py) alone does not add
1797 # them — slowapi only injects headers when a caller invokes
1798 # `_inject_headers` itself, which its own default
1799 # `_rate_limit_exceeded_handler` (slowapi/extension.py) does; this
1800 # app registers a custom handler instead (for the audit log line
1801 # above and main's ``{"error", "message"}`` body contract) and,
1802 # before this fix, never called it, so live 429s carried none of
1803 # these headers even with the flag on. Flask-Limiter added them on
1804 # main via its own ``after_request`` hook — no direct FastAPI
1805 # equivalent exists, so it has to happen here, per-handler.
1806 #
1807 # Uses the `limiter` closed over above (same object as
1808 # `app.state.limiter`) rather than going through
1809 # `request.app.state.limiter` — the two are identical for the real
1810 # app, but the latter forces every caller of this handler to have
1811 # a real `request.app`, and tests/web/test_rate_limit_coverage.py
1812 # ::TestRateLimitExceededHandler calls this handler directly with
1813 # a bare ``Mock()`` request (pinning the audit-log/body contract in
1814 # isolation, unrelated to this fix). Going through `request.app`
1815 # there silently replaces `response` with an auto-generated Mock
1816 # instead of exercising real header-injection logic.
1817 #
1818 # `request.state.view_rate_limit` is set by slowapi's
1819 # `Limiter.__evaluate_limits` (private method) immediately before
1820 # it raises `RateLimitExceeded` — verified against the installed
1821 # slowapi source (`.venv/lib/*/site-packages/slowapi/__init__.py`)
1822 # rather than assumed, since both the attribute and
1823 # `_inject_headers` are private slowapi API that could rename
1824 # across versions. The whole call is wrapped in a try/except
1825 # (rather than trusting `_inject_headers`'s own internal
1826 # swallow_errors — this Limiter is built with
1827 # `swallow_errors=False`, i.e. it re-raises) so nothing about this
1828 # best-effort header injection — a missing `view_rate_limit`, a
1829 # storage hiccup, or (as in the Mock-request test above) a
1830 # request.state that doesn't behave like a real one — can ever
1831 # turn a 429 into a 500. On any failure `response` simply keeps
1832 # its original value, assigned before the `try`.
1833 #
1834 # tests/web/test_rate_limit_headers_on_429.py pins this against a
1835 # real 429 from a real rate-limited route so a future slowapi
1836 # rename of either private symbol fails loudly here instead of
1837 # silently dropping the headers again.
1838 # Set the headers directly rather than calling
1839 # `limiter._inject_headers`. That method is gated on the Limiter's
1840 # `_headers_enabled`, and this app deliberately leaves that flag OFF
1841 # — turning it on makes slowapi wrap EVERY rate-limited route and
1842 # raise on any handler that returns a plain dict instead of a
1843 # Response, turning ordinary 200s into 500s (see the comment on
1844 # `_limiter_kwargs` in dependencies/rate_limit.py). So calling
1845 # `_inject_headers` here would silently no-op; the values have to be
1846 # computed here instead.
1847 try:
1848 current_limit = getattr(request.state, "view_rate_limit", None)
1849 if current_limit is not None:
1850 item, key_parts = current_limit
1851 reset_at, remaining = limiter.limiter.get_window_stats(
1852 item, *key_parts
1853 )
1854 response.headers["Retry-After"] = str(
1855 max(0, int(reset_at - time.time()))
1856 )
1857 response.headers["X-RateLimit-Limit"] = str(item.amount)
1858 response.headers["X-RateLimit-Remaining"] = str(remaining)
1859 response.headers["X-RateLimit-Reset"] = str(int(reset_at))
1860 except Exception:
1861 logger.debug(
1862 "Could not attach Retry-After/X-RateLimit-* headers to the "
1863 "429 response; returning it without them."
1864 )
1865 return response
1867 app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded)
1869 # Enforce the global DEFAULT_RATE_LIMIT on every route without an
1870 # explicit limit decorator (decorated routes are exempt here — their
1871 # decorators handle their own checks; mounts like /ws have no route
1872 # handler and are skipped). Flask-Limiter's default_limits behaved
1873 # the same way on main.
1874 app.add_middleware(SlowAPIMiddleware)
1877# ---------------------------------------------------------------------------
1878# Static file serving
1879# ---------------------------------------------------------------------------
1881_HASHED_FILENAME_RE = re.compile(r"\.[A-Za-z0-9_-]{8,}\.")
1884def _add_static_routes(app: FastAPI) -> None:
1885 """Add static file serving routes with cache control."""
1886 from ..security.path_validator import PathValidator
1888 static_dir = Path(STATIC_DIR)
1889 dist_dir = static_dir / "dist"
1891 # Static assets must not share the API's default per-IP bucket: base.html
1892 # alone pulls ~29 assets, served must-revalidate, so a few page loads
1893 # could exhaust a user's whole hourly budget. main exempted these.
1894 from .dependencies.rate_limit import limiter as _limiter
1896 @app.get("/favicon.ico", include_in_schema=False)
1897 @_limiter.exempt
1898 async def favicon():
1899 from fastapi.responses import FileResponse
1901 favicon_path = static_dir / "favicon.ico"
1902 if favicon_path.exists():
1903 return FileResponse(str(favicon_path), media_type="image/x-icon")
1904 return JSONResponse({"error": "Not found"}, status_code=404)
1906 @app.get("/static/{path:path}", include_in_schema=False)
1907 @_limiter.exempt
1908 async def serve_static(path: str):
1909 from fastapi.responses import FileResponse
1911 # Try dist directory first (Vite-built assets)
1912 dist_prefix = "dist/"
1913 if path.startswith(dist_prefix):
1914 rel_path = path[len(dist_prefix) :]
1915 try:
1916 validated = PathValidator.validate_safe_path(
1917 rel_path, dist_dir, allow_absolute=False
1918 )
1919 if validated and validated.is_file(): 1919 ↛ 1939line 1919 didn't jump to line 1939 because the condition on line 1919 was always true
1920 headers = {}
1921 if _HASHED_FILENAME_RE.search(rel_path):
1922 headers["Cache-Control"] = (
1923 "public, max-age=31536000, immutable"
1924 )
1925 else:
1926 headers["Cache-Control"] = (
1927 "public, max-age=0, must-revalidate"
1928 )
1929 return FileResponse(str(validated), headers=headers)
1930 except (ValueError, OSError):
1931 # OSError too: is_file() stats the path, so a segment longer
1932 # than the filesystem's NAME_MAX raises ENAMETOOLONG (errno
1933 # 36), which `except ValueError` does not catch. This route
1934 # is unauthenticated AND rate-limit exempt, so an uncaught
1935 # one is an anonymous 500 for any long path.
1936 pass
1938 # Try dist directory for Vite assets (fonts, etc.)
1939 try:
1940 validated = PathValidator.validate_safe_path(
1941 path, dist_dir, allow_absolute=False
1942 )
1943 if validated and validated.is_file():
1944 headers = {}
1945 if _HASHED_FILENAME_RE.search(path): 1945 ↛ 1946line 1945 didn't jump to line 1946 because the condition on line 1945 was never true
1946 headers["Cache-Control"] = (
1947 "public, max-age=31536000, immutable"
1948 )
1949 else:
1950 headers["Cache-Control"] = (
1951 "public, max-age=0, must-revalidate"
1952 )
1953 return FileResponse(str(validated), headers=headers)
1954 except (ValueError, OSError):
1955 # is_file() can raise OSError (ENAMETOOLONG), not just
1956 # ValueError -- see the note on the first handler above.
1957 pass
1959 # Fall back to regular static directory
1960 try:
1961 validated = PathValidator.validate_safe_path(
1962 path, static_dir, allow_absolute=False
1963 )
1964 if validated and validated.is_file():
1965 return FileResponse(
1966 str(validated),
1967 headers={
1968 "Cache-Control": "public, max-age=0, must-revalidate"
1969 },
1970 )
1971 except (ValueError, OSError):
1972 # is_file() can raise OSError (ENAMETOOLONG), not just
1973 # ValueError -- see the note on the first handler above.
1974 pass
1976 return JSONResponse({"error": "Not found"}, status_code=404)
1979# ---------------------------------------------------------------------------
1980# Template helpers (replaces Flask's context_processor / jinja_env.globals)
1981# ---------------------------------------------------------------------------
1983# Matches url_for('name') / url_for("name") usages in templates. Shared by
1984# _validate_url_for_bindings (below) and
1985# tests/web/templates/test_url_for_links.py, which independently extracts
1986# the same names to fence dead links.
1987_URL_FOR_NAME_RE = re.compile(r"url_for\(\s*['\"]([^'\"]+)['\"]")
1990def _setup_template_globals() -> None:
1991 """Register template globals for Jinja2 (CSRF, url_for, flash, vite, themes, etc.)."""
1992 env = templates.env
1994 # Vite asset helpers (must be registered at module level, not in lifespan)
1995 from .utils.vite_helper import vite
1997 vite.init_for_fastapi(STATIC_DIR, templates)
1999 # CSRF token — needs request, so we provide a callable that reads from
2000 # the template context (where request is passed per-render).
2001 # Templates call {{ csrf_token() }} which reads _csrf_request from context.
2003 def csrf_token_factory():
2004 """Returns empty string by default. Overridden per-request via context."""
2005 return ""
2007 env.globals["csrf_token"] = csrf_token_factory
2009 # Flash messages — same pattern: needs request.
2010 env.globals["get_flashed_messages"] = lambda with_categories=False: []
2012 # Theme helpers
2013 from .themes import get_theme_metadata, get_themes, get_themes_json
2015 env.globals["get_themes"] = get_themes
2016 env.globals["get_themes_json"] = get_themes_json
2017 env.globals["get_theme_metadata"] = get_theme_metadata
2019 # Session — default empty dict, overridden per-request via context
2020 env.globals["session"] = {}
2022 # Egress scope — default to the registered fallback so templates
2023 # rendered WITHOUT render_template() (e.g. the index route's direct
2024 # TemplateResponse) can still reference egress_scope on
2025 # <body data-scope=…> without raising an UndefinedError. render_template()
2026 # overrides this per-render with the user's resolved scope.
2027 from ..security.egress.policy import DEFAULT_EGRESS_SCOPE
2029 env.globals["egress_scope"] = DEFAULT_EGRESS_SCOPE
2031 # Kept INSIDE this function as a single direct dict literal, not hoisted
2032 # to module scope: .pre-commit-hooks/check-url-for-targets.py parses this
2033 # function statically and requires exactly one literal map here ("Expected
2034 # one direct literal map"). _validate_url_for_bindings does not need the
2035 # dict — it resolves names through templates.env.globals["url_for"], the
2036 # same closure the templates use.
2037 # url_for wrapper map (Flask-style blueprint.endpoint -> path).
2038 _URL_MAP = {
2039 # Auth
2040 "auth.login": "/auth/login",
2041 "auth.login_page": "/auth/login",
2042 "auth.register": "/auth/register",
2043 "auth.register_page": "/auth/register",
2044 "auth.logout": "/auth/logout",
2045 "auth.change_password": "/auth/change-password",
2046 "auth.change_password_page": "/auth/change-password",
2047 # Pages
2048 "index": "/",
2049 "static": "/static",
2050 "history.history_page": "/history/",
2051 "chat.chat_page": "/chat/",
2052 # Library / RAG
2053 "library.library_page": "/library/",
2054 "library.download_manager_page": "/library/download-manager",
2055 "rag.collections_page": "/library/collections",
2056 "rag.embedding_settings_page": "/library/embedding-settings",
2057 # Zotero page lives under the /library prefix; without this the
2058 # dot-to-slash fallback yields a dead /zotero/zotero_page link.
2059 "zotero.zotero_page": "/library/zotero",
2060 # Notes + unified-search pages (the dot-to-slash fallback would
2061 # otherwise produce /notes/notes_page and
2062 # /unified_search/unified_search_page dead links).
2063 "notes.notes_page": "/notes/",
2064 "unified_search.unified_search_page": "/library/search/",
2065 # News
2066 "news.news_page": "/news/",
2067 "news.subscriptions_page": "/news/subscriptions",
2068 # Metrics
2069 "metrics.metrics_dashboard": "/metrics/",
2070 "metrics.journal_quality": "/metrics/journals",
2071 # Benchmark
2072 "benchmark.index": "/benchmark/",
2073 # Currently also produced by the dot-to-slash fallback, but pin it
2074 # so a rename of the route function can't silently break the
2075 # sidebar link.
2076 "benchmark.results": "/benchmark/results",
2077 # Settings (page + form action)
2078 "settings.settings_page": "/settings/",
2079 "settings.save_settings": "/settings/save_settings",
2080 "settings.save_all_settings": "/settings/save_all_settings",
2081 "settings.main_config_page": "/settings/main",
2082 "settings.collections_config_page": "/settings/collections",
2083 "settings.api_keys_config_page": "/settings/api_keys",
2084 "settings.llm_config_page": "/settings/llm",
2085 "settings.search_engines_config_page": "/settings/search_engines",
2086 }
2088 def url_for(name, **kwargs):
2089 path = _URL_MAP.get(name, f"/{name.replace('.', '/')}")
2090 if name == "static" and "filename" in kwargs:
2091 path = f"/static/{kwargs['filename']}"
2092 elif name == "static" and "path" in kwargs: 2092 ↛ 2093line 2092 didn't jump to line 2093 because the condition on line 2092 was never true
2093 path = f"/static/{kwargs['path']}"
2094 # Append query params
2095 query_params = {
2096 k: v for k, v in kwargs.items() if k not in ("filename", "path")
2097 }
2098 if query_params:
2099 from urllib.parse import urlencode
2101 path += "?" + urlencode(query_params)
2102 return path
2104 env.globals["url_for"] = url_for
2107# ---------------------------------------------------------------------------
2108# Create the FastAPI application
2109# ---------------------------------------------------------------------------
2111# Deliberately does NOT check the generic `CI` / `TESTING` variables, which an
2112# earlier version did. This flag suppresses the `Secure` attribute on session
2113# cookies (SecureCookieMiddleware) *and* both operator warnings about serving
2114# a public instance over plain HTTP, so any CD pipeline or PaaS that exports
2115# CI=true would have shipped session cookies without `Secure` over HTTPS, and
2116# silenced the warning that would have said so.
2117#
2118# This is the same reasoning already written down in
2119# database/sqlcipher_utils.py::_get_min_kdf_iterations, which refuses to relax
2120# the KDF floor on CI/TESTING for exactly this reason. The cookie path had not
2121# been given that treatment.
2122#
2123# PYTEST_CURRENT_TEST is presence-based (pytest sets it to a descriptive
2124# string); LDR_TEST_MODE is parsed as a real boolean so `LDR_TEST_MODE=0` does
2125# not enable test mode.
2126#
2127# CAVEAT, measured rather than assumed: this is evaluated at IMPORT time, and
2128# pytest sets PYTEST_CURRENT_TEST only while a test body runs -- not during
2129# collection, which is when a test module's top-level `import fastapi_app`
2130# actually happens. So under a normal run this half is False and
2131# LDR_TEST_MODE is the only switch that reaches here. Contrast
2132# sqlcipher_utils::_get_min_kdf_iterations, which reads the same variable
2133# INSIDE a function and therefore does see it.
2134#
2135# That is deliberate rather than a gap worth "fixing": the effect of False is
2136# that tests exercise the production cookie path, which is the stricter and
2137# more realistic behaviour. Do not switch this to a lazy/per-request lookup
2138# expecting tests to relax -- it would re-open exactly the hole the paragraph
2139# above describes, just keyed on a different variable.
2140is_testing = bool(os.getenv("PYTEST_CURRENT_TEST")) or to_bool(
2141 os.getenv("LDR_TEST_MODE")
2142)
2144# Public API docs are disabled by default — a multi-user deployment
2145# shouldn't expose the full schema unauthenticated. Set LDR_EXPOSE_DOCS=true
2146# to re-enable (useful for internal/dev instances).
2147_expose_docs = os.getenv("LDR_EXPOSE_DOCS", "").lower() in ("true", "1", "yes")
2148_docs_enabled = _expose_docs and not is_testing
2150app = FastAPI(
2151 title="Local Deep Research",
2152 version=__version__,
2153 docs_url="/api/docs" if _docs_enabled else None,
2154 # Gate the OpenAPI schema endpoint together with the Swagger UI:
2155 # /openapi.json IS the full API schema, so leaving it on its default
2156 # "/openapi.json" while disabling docs_url still exposes the entire
2157 # surface unauthenticated. app.openapi() remains callable in-process.
2158 openapi_url="/openapi.json" if _docs_enabled else None,
2159 redoc_url=None,
2160 lifespan=lifespan,
2161)
2163# FastAPI runs plain ``def`` endpoints on AnyIO workers, whereas ASGI
2164# middleware teardown runs on the event-loop thread. Install an endpoint
2165# wrapper so thread-local DB sessions are released on the worker that owns
2166# them. Populated module routers receive the same dispatch wrapper in
2167# ``_mount_all`` without replacing their registered endpoint objects.
2168from .dependencies.threadpool import WorkerCleanupAPIRoute # noqa: E402
2170app.router.route_class = WorkerCleanupAPIRoute
2172# ASGI middleware stack.
2173# Starlette add_middleware is LIFO: last added = outermost.
2174# Desired order (outer→inner): SecureCookie → SecurityHeaders → Session → CSRF → Database → app
2175#
2176# So we add in reverse: Database first, then CSRF, then Session, etc.
2178app.add_middleware(DatabaseMiddleware)
2180# CSRF runs after Session populates scope["session"] and rejects state-changing
2181# requests with a missing/invalid X-CSRFToken header before they reach handlers.
2182from .dependencies.csrf import CSRFMiddleware # noqa: E402
2184app.add_middleware(CSRFMiddleware)
2186app.add_middleware(
2187 SessionMiddleware,
2188 secret_key=SECRET_KEY,
2189 session_cookie="session",
2190 max_age=get_security_default("security.session_remember_me_days", 30)
2191 * 24
2192 * 3600,
2193 # "strict" is safe here because LDR has no OAuth redirects or
2194 # external flows that require cross-site cookie delivery.
2195 same_site="strict",
2196 https_only=False, # Handled dynamically by SecureCookieMiddleware
2197)
2199# Runs just outside SessionMiddleware so it sees the Set-Cookie
2200# header SessionMiddleware just wrote, and can strip Max-Age/Expires
2201# for non-remember-me logins. See the class docstring.
2202app.add_middleware(RememberMeMiddleware)
2204# Body cap sits inside SecurityHeaders so its 413 responses still get
2205# security headers, but outside Session/CSRF so over-limit bodies are
2206# rejected before anything buffers them.
2207app.add_middleware(BodySizeLimitMiddleware)
2209app.add_middleware(SecurityHeadersMiddleware)
2210app.add_middleware(SecureCookieMiddleware, testing=is_testing)
2213# Prefixes that carried CORS headers on main. `SecurityHeaders._is_api_route`
2214# (security/security_headers.py, deleted by the FastAPI migration) gated CORS to
2215# exactly these three, so an operator who whitelisted an origin for API access
2216# did not thereby grant it cross-origin reads of every HTML page. Restored here.
2217_CORS_API_PREFIXES = ("/api/", "/research/api/", "/history/api")
2220class _PathScopedCORSMiddleware:
2221 """Apply Starlette's CORSMiddleware only to API paths.
2223 Starlette's CORSMiddleware has no path predicate: mounted app-wide it
2224 answers preflights and stamps `Access-Control-Allow-Origin` on every route,
2225 HTML pages included. main scoped that to three prefixes and the migration
2226 dropped the scoping along with `_is_api_route`.
2228 Pure ASGI rather than BaseHTTPMiddleware, deliberately: this sits in front
2229 of the streaming SSE routes, and BaseHTTPMiddleware buffers
2230 StreamingResponse bodies (the defect already tracked for
2231 SlowAPIMiddleware). Non-matching paths are handed to the inner app
2232 untouched, so the pass-through case adds one string comparison and no
2233 wrapping at all.
2235 Non-HTTP scopes (websocket, lifespan) bypass CORS entirely — the CORS
2236 protocol is HTTP-only, and the Socket.IO mount does its own origin check.
2237 """
2239 def __init__(self, app, prefixes, cors_factory):
2240 self.app = app
2241 self.prefixes = tuple(prefixes)
2242 # Build the real CORSMiddleware once, wrapping the inner app, so a
2243 # matching request gets Starlette's full preflight/origin handling.
2244 self.cors_app = cors_factory(app)
2246 async def __call__(self, scope, receive, send):
2247 if scope["type"] != "http":
2248 await self.app(scope, receive, send)
2249 return
2250 path = scope.get("path", "")
2251 if path.startswith(self.prefixes):
2252 await self.cors_app(scope, receive, send)
2253 else:
2254 await self.app(scope, receive, send)
2257def _configure_cors(app: FastAPI) -> None:
2258 """Restore configurable cross-origin access for API clients.
2260 Ports the CORS support main applied in security_headers.py from
2261 ``security.cors.allowed_origins`` (env: LDR_SECURITY_CORS_ALLOWED_ORIGINS),
2262 which the FastAPI migration dropped — the setting was live config on
2263 main but reached no code on this branch. Uses Starlette's
2264 CORSMiddleware (correct preflight/OPTIONS handling, origin matching)
2265 rather than hand-injecting headers.
2267 Fail closed: with the setting empty (the default) no CORS middleware
2268 is registered, so the app stays same-origin only — exactly main's
2269 empty-default behavior. Same-origin requests carry no Origin header
2270 and never receive CORS headers regardless, so this is effectively
2271 scoped to genuine cross-origin API calls.
2272 """
2273 from ..settings.env_registry import get_env_setting
2275 configured = get_env_setting("security.cors.allowed_origins")
2276 if not configured:
2277 return
2279 if configured.strip() == "*":
2280 # Credentials with a wildcard origin is forbidden by the CORS spec
2281 # (and was a startup error on main). Allow the wildcard for
2282 # non-credentialed access only.
2283 origins = ["*"]
2284 allow_credentials = False
2285 else:
2286 origins = [o.strip() for o in configured.split(",") if o.strip()]
2287 allow_credentials = False
2289 from starlette.middleware.cors import CORSMiddleware
2291 app.add_middleware(
2292 _PathScopedCORSMiddleware,
2293 prefixes=_CORS_API_PREFIXES,
2294 cors_factory=lambda inner: CORSMiddleware(
2295 inner,
2296 allow_origins=origins,
2297 allow_credentials=allow_credentials,
2298 allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
2299 allow_headers=[
2300 "Content-Type",
2301 "Authorization",
2302 "X-Requested-With",
2303 "X-HTTP-Method-Override",
2304 "X-CSRFToken",
2305 ],
2306 max_age=3600,
2307 ),
2308 )
2309 logger.info(
2310 f"CORS enabled for API origins: {origins} "
2311 f"(scoped to {', '.join(_CORS_API_PREFIXES)})"
2312 )
2315# Rate limiting via slowapi + exception handlers
2316_setup_rate_limiting(app)
2317_register_exception_handlers(app)
2319# Registered last of the request-handling middleware, so it is the outermost
2320# of them: a preflight OPTIONS is answered before auth, CSRF, or rate limiting
2321# ever run. (The CI-only request-timing layer below is added after it and so
2322# sits further out still, but it only logs and passes every scope through, so
2323# it changes nothing about that ordering.)
2324_configure_cors(app)
2326# CI/test-only request forensics for the #4431 navigation-stall hunts: logs
2327# every request's arrival + duration so a silent log window can be attributed
2328# to "request never arrived" vs "request hung in the app". Ports #4536, which
2329# merge 5ad5f5a1b reverted along with app_factory.py (#5959). Added LAST of
2330# all `add_middleware` calls because that list is LIFO — last added is
2331# outermost — which is the only position from which it measures true
2332# arrival-to-response time.
2333if _request_timing_enabled(): 2333 ↛ 2334line 2333 didn't jump to line 2334 because the condition on line 2333 was never true
2334 app.add_middleware(RequestTimingASGIMiddleware)
2335 logger.info("Request-timing forensics middleware enabled (CI/TESTING)")
2337# Static file routes
2338_add_static_routes(app)
2340# Template globals
2341_setup_template_globals()
2343# ---------------------------------------------------------------------------
2344# Mount routers and Socket.IO (deferred imports to avoid E402)
2345# ---------------------------------------------------------------------------
2348def _mount_all(app: FastAPI) -> None:
2349 """Mount all routers and Socket.IO on the app."""
2350 from ..database.session_context import get_user_db_session
2351 from ..settings.logger import log_settings
2352 from ..utilities.db_utils import get_settings_manager
2353 from .dependencies.threadpool import prepare_router_for_worker_cleanup
2354 from .routers.api_v1 import router as api_v1_router
2355 from .services.socketio_asgi import socket_app
2357 # Mount Socket.IO
2358 app.mount("/ws", socket_app)
2360 # Include all routers — error-tolerant loading during migration.
2361 # Routers with unresolved import issues will be skipped with a warning.
2362 _routers = {
2363 "api_v1": api_v1_router,
2364 }
2366 # Phase 2+ routers — Phase 2-7 is complete; an import error here is a
2367 # production-class regression, not a migration-in-progress signal. Fail
2368 # loud at startup instead of silently serving 404 for the whole router.
2369 _router_modules = [
2370 ("auth", ".routers.auth"),
2371 ("research", ".routers.research"),
2372 ("history", ".routers.history"),
2373 ("settings", ".routers.settings"),
2374 ("metrics", ".routers.metrics"),
2375 ("api", ".routers.api"),
2376 ("context_overflow", ".routers.context_overflow_api"),
2377 ("news_api", ".routers.news_flask_api"),
2378 ("news_pages", ".routers.news_pages"),
2379 ("benchmark", ".routers.benchmark"),
2380 ("followup", ".routers.followup"),
2381 ("library", ".routers.library"),
2382 ("rag", ".routers.rag"),
2383 ("library_delete", ".routers.library_delete"),
2384 ("library_search", ".routers.library_search"),
2385 ("zotero", ".routers.zotero"),
2386 ("notes", ".routers.notes"),
2387 ("unified_search", ".routers.unified_search"),
2388 ("scheduler", ".routers.scheduler"),
2389 ("chat", ".routers.chat"),
2390 ]
2392 import importlib
2394 for name, module_path in _router_modules:
2395 mod = importlib.import_module(module_path, package=__package__)
2396 if not hasattr(mod, "router"):
2397 raise RuntimeError(
2398 f"Router module '{module_path}' has no 'router' attribute"
2399 )
2400 _routers[name] = mod.router
2402 for name, r in _routers.items():
2403 app.include_router(r)
2405 # include_router() preserves the source APIRoute class rather than using
2406 # app.router.route_class for already-populated module routers. Apply the
2407 # cleanup to the copied dispatch targets after every router is mounted;
2408 # route.endpoint remains the original handler for introspection, SlowAPI,
2409 # and route-identity security checks.
2410 prepare_router_for_worker_cleanup(app.router)
2412 @app.get("/", include_in_schema=False)
2413 def index(request: Request):
2414 """Root route - redirect to login if not authenticated."""
2415 username = request.session.get("username")
2416 if not username:
2417 return RedirectResponse(url="/auth/login", status_code=302)
2419 from ..constants import get_available_strategies
2420 from ..security.egress.policy import (
2421 DEFAULT_EGRESS_SCOPE,
2422 effective_scope_for_display,
2423 unprotected_egress_allowed,
2424 )
2425 from ..settings.manager import check_env_setting
2427 settings = {}
2428 try:
2429 with get_user_db_session(username) as db_session:
2430 if db_session: 2430 ↛ 2517line 2430 didn't jump to line 2517
2431 sm = get_settings_manager(db_session, username)
2432 # Operator-gated escape hatch (#5148 / 87537d9ec): the
2433 # raw stored scope may be a legacy/invalid value or an
2434 # "unprotected" selection the operator has since
2435 # disabled — normalise what we DISPLAY through the same
2436 # helper the policy engine uses so the UI never shows a
2437 # scope the run-time layer would reject. allow_
2438 # unprotected_egress gates the <option value="unprotected">
2439 # entry in research.html's select; the *_env_locked flags
2440 # let it grey out controls the operator pinned via env
2441 # vars (server-side overrides are enforced independently
2442 # in routers/research.py::_apply_policy_overrides).
2443 raw_egress_scope = sm.get_setting(
2444 "policy.egress_scope", DEFAULT_EGRESS_SCOPE
2445 )
2446 effective_egress_scope = effective_scope_for_display(
2447 raw_egress_scope
2448 )
2449 settings = {
2450 "llm_provider": sm.get_setting(
2451 "llm.provider", "ollama"
2452 ),
2453 "llm_model": sm.get_setting("llm.model", ""),
2454 "llm_openai_endpoint_url": sm.get_setting(
2455 "llm.openai_endpoint.url", ""
2456 ),
2457 "llm_ollama_url": sm.get_setting("llm.ollama.url"),
2458 "llm_lmstudio_url": sm.get_setting("llm.lmstudio.url"),
2459 "llm_local_context_window_size": sm.get_setting(
2460 "llm.local_context_window_size"
2461 ),
2462 "search_tool": sm.get_setting("search.tool", ""),
2463 "search_iterations": sm.get_setting(
2464 "search.iterations", 3
2465 ),
2466 "search_questions_per_iteration": sm.get_setting(
2467 "search.questions_per_iteration", 2
2468 ),
2469 "search_strategy": sm.get_setting(
2470 "search.search_strategy", "source-based"
2471 ),
2472 # Egress-policy controls (Stage 1c UI) — main loads
2473 # these so the research page reflects the saved policy
2474 # instead of silently defaulting (fail-open).
2475 "policy_egress_scope": effective_egress_scope,
2476 "egress_scope_env_locked": check_env_setting(
2477 "policy.egress_scope"
2478 )
2479 is not None,
2480 "allow_unprotected_egress": unprotected_egress_allowed(),
2481 "llm_local_env_locked": check_env_setting(
2482 "llm.require_local_endpoint"
2483 )
2484 is not None,
2485 "embeddings_local_env_locked": check_env_setting(
2486 "embeddings.require_local"
2487 )
2488 is not None,
2489 "llm_require_local_endpoint": sm.get_setting(
2490 "llm.require_local_endpoint", False
2491 ),
2492 "embeddings_require_local": sm.get_setting(
2493 "embeddings.require_local", False
2494 ),
2495 }
2496 except Exception:
2497 # Password not available (e.g. server restarted) — force re-login.
2498 #
2499 # Log before redirecting. This handler wraps ~70 lines (session
2500 # acquisition, settings manager, egress scope, and 18 get_setting
2501 # calls) on the app's ROOT route, so it catches far more than the
2502 # documented password case — a bug in any of those reads presents
2503 # to the user as "I keep getting randomly logged out" and left no
2504 # trace at any log level. Flask had no try/except here at all
2505 # (web/app_factory.py's index()), so the same fault produced a
2506 # logged 500; swallowing it silently is a diagnosability
2507 # regression, not a behaviour improvement. The redirect itself is
2508 # correct for the case named above and is kept.
2509 logger.exception(
2510 "Root route failed to load settings for user {} — clearing "
2511 "session and redirecting to login",
2512 request.session.get("username"),
2513 )
2514 request.session.clear()
2515 return RedirectResponse(url="/auth/login", status_code=302)
2517 log_settings(settings, "Research page settings loaded")
2519 return templates.TemplateResponse(
2520 request=request,
2521 name="pages/research.html",
2522 context={
2523 "settings": settings,
2524 "strategies": get_available_strategies(),
2525 },
2526 )
2529_mount_all(app)
2532# ---------------------------------------------------------------------------
2533# Startup validation: every url_for() name used by a template must resolve
2534# to a route actually mounted on the app.
2535# ---------------------------------------------------------------------------
2538def _validate_url_for_bindings(app: "FastAPI") -> None:
2539 """Fail loudly at import/startup time if the Flask-compat ``url_for``
2540 shim would silently hand any template a dead link.
2542 ``url_for``'s fallback (``f"/{name.replace('.', '/')}"``) produces a
2543 plausible-looking but wrong URL for any name missing from ``_URL_MAP``
2544 — that already shipped one dead link (``chat.chat_page`` ->
2545 ``/chat/chat_page``, see ``tests/web/templates/test_url_for_links.py``).
2546 Rather than raise inside ``url_for()`` at render time (which would turn
2547 one cosmetic bad link into a 500 for an otherwise-working page — a worse
2548 failure mode for a self-hosted app), this scans every template for
2549 ``url_for("name")`` usages, resolves each one exactly as the real shim
2550 would, and checks the RESOLVED path against the live route table. This
2551 must run AFTER ``_mount_all(app)`` -- ``app.routes`` is empty at the
2552 point ``_setup_template_globals()`` runs (before routers are mounted),
2553 so validating there would trivially "pass" with an unpopulated table.
2555 Deliberately does NOT restrict itself to `_URL_MAP` keys: the
2556 `chat.chat_page` bug was a MISSING entry falling through to the
2557 dot-to-slash guess, which key-only validation would never have caught.
2559 Matching strategy: a resolved path is valid if EITHER
2560 (a) it exactly equals the ``path`` of some plain (parameter-free)
2561 ``Route`` — with a trailing-slash-tolerant comparison, since pages
2562 are linked both with and without one, or
2563 (b) it falls under the prefix of something that is *designed* to
2564 swallow an arbitrary nested sub-path: an ``app.mount(...)``
2565 (``Mount`` — e.g. the Socket.IO ASGI app at ``/ws``) or a ``Route``
2566 using Starlette's catch-all ``{name:path}`` converter (e.g.
2567 ``serve_static``'s ``/static/{path:path}``, which is exactly how
2568 ``url_for("static", filename=...)`` is validated without a special
2569 case).
2571 Every OTHER parameterized route (e.g. ``/chat/{session_id}``, a plain
2572 string converter bound to one specific, known value) is intentionally
2573 EXCLUDED from matching, even though Starlette's own ``Route.matches()``
2574 would happily match any string there. That was tried first and
2575 rejected: ``/chat/{session_id}`` matches literally any string under
2576 ``/chat/``, including ``/chat/totally-wrong-path`` — the exact shape of
2577 wrong path a missing ``_URL_MAP`` entry produces — so a
2578 ``Route.matches()``-based check would NOT have caught the
2579 ``chat.chat_page`` regression this validator exists to prevent.
2580 """
2581 from starlette.routing import Mount, Route
2583 url_for = templates.env.globals["url_for"]
2585 template_dir = Path(TEMPLATE_DIR)
2586 names: set[str] = set()
2587 for tpl_path in template_dir.rglob("*.html"):
2588 names.update(
2589 _URL_FOR_NAME_RE.findall(
2590 tpl_path.read_text(encoding="utf-8", errors="replace")
2591 )
2592 )
2594 if not names:
2595 # Deliberately NOT fatal. An empty scan is indistinguishable from a
2596 # packaging shape: TEMPLATE_DIR is computed from
2597 # importlib_resources.as_file(), whose temp directory is removed when
2598 # its context manager exits, so a zip-imported / pex / frozen install
2599 # can legitimately have no readable templates directory here.
2600 # Jinja2Templates itself tolerates that (FileSystemLoader never checks
2601 # the directory exists), so the pre-existing behaviour is "HTML pages
2602 # fail, the API keeps serving". Raising here would escalate that to
2603 # "the process does not import at all", which is strictly worse and is
2604 # exactly the trade this function's own docstring rejects.
2605 logger.warning(
2606 "url_for startup validation found no url_for() usages under "
2607 f"{template_dir}; skipping link validation. If this is a normal "
2608 "source install the templates directory has moved, and HTML "
2609 "rendering is already broken."
2610 )
2611 return
2613 static_paths: set[str] = set()
2614 wildcard_prefixes: set[str] = set()
2615 _catchall_re = re.compile(r"\{[^{}:]+:path\}")
2616 for route in app.routes:
2617 path = getattr(route, "path", None)
2618 if not path: 2618 ↛ 2619line 2618 didn't jump to line 2619 because the condition on line 2618 was never true
2619 continue
2620 if isinstance(route, Mount):
2621 wildcard_prefixes.add(path.rstrip("/"))
2622 elif isinstance(route, Route): 2622 ↛ 2616line 2622 didn't jump to line 2616 because the condition on line 2622 was always true
2623 if "{" not in path:
2624 static_paths.add(path)
2625 static_paths.add(path.rstrip("/") or "/")
2626 elif _catchall_re.search(path):
2627 wildcard_prefixes.add(path.split("{", 1)[0].rstrip("/"))
2628 # else: parameterized on a specific value (e.g. {session_id}) --
2629 # not a wildcard bucket url_for is allowed to land in.
2631 def _under_wildcard_prefix(lookup: str) -> bool:
2632 return any(
2633 lookup == prefix or lookup.startswith(prefix + "/")
2634 for prefix in wildcard_prefixes
2635 )
2637 dead: list[str] = []
2638 for name in sorted(names):
2639 path = (
2640 url_for(name, filename="x") if name == "static" else url_for(name)
2641 )
2642 # Normalise BOTH sides. static_paths stores stripped and
2643 # unstripped forms, but without stripping the lookup the
2644 # tolerance was one-directional: nine _URL_MAP values end in "/"
2645 # and pass only because each of those routers happens to declare
2646 # @router.get("/"). Changing one to @router.get("") — invisible
2647 # at runtime thanks to redirect_slashes — would have bricked boot.
2648 lookup = path.split("?", 1)[0]
2649 lookup = lookup.rstrip("/") or "/"
2650 if lookup in static_paths or _under_wildcard_prefix(lookup):
2651 continue
2652 dead.append(f"{name!r} -> {path!r}")
2654 if dead:
2655 detail = (
2656 "the following template url_for() names resolve to paths that "
2657 "are not mounted on the app. Add/fix an entry in _URL_MAP "
2658 "(web/fastapi_app.py) for each:\n " + "\n ".join(dead)
2659 )
2660 # Strict only when asked. A dead nav link is a cosmetic defect; making
2661 # it un-bootable turns one broken link into total unavailability,
2662 # which is a worse failure mode than the render-time raise this
2663 # function already rejects for the same reason. CI and developers get
2664 # the hard failure via LDR_STRICT_TEMPLATE_LINKS and via
2665 # tests/web/templates/test_url_for_links.py, which asserts the same
2666 # property offline; a running deployment gets a loud log and keeps
2667 # serving.
2668 if os.environ.get("LDR_STRICT_TEMPLATE_LINKS", "").lower() in (
2669 "1",
2670 "true",
2671 "yes",
2672 ):
2673 raise RuntimeError("url_for() startup validation failed: " + detail)
2674 logger.error("url_for() startup validation failed: " + detail)
2677_validate_url_for_bindings(app)