Coverage for src/local_deep_research/web/fastapi_app.py: 92%
888 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2FastAPI application factory for Local Deep Research.
4This replaces Flask's app_factory.py as the primary web application entry point.
5It mounts Socket.IO as an ASGI sub-app, configures middleware, templates,
6static files, and background services.
7"""
9import ipaddress
10import json
11import logging
12import os
13import re
14import secrets
15import sys
16import time
17from contextlib import asynccontextmanager
18from importlib import resources as importlib_resources
19from pathlib import Path
21from fastapi import FastAPI, HTTPException, Request
22from fastapi.responses import (
23 HTMLResponse,
24 JSONResponse,
25 PlainTextResponse,
26 RedirectResponse,
27)
28from loguru import logger
29from starlette.middleware.sessions import SessionMiddleware
30from starlette.requests import ClientDisconnect
32from ..__version__ import __version__
33from ..security import get_security_default
34from ..security.network_utils import is_ipv6_transition_address
35from ..utilities.type_utils import to_bool
36from ..utilities.log_utils import InterceptHandler
38# ---------------------------------------------------------------------------
39# Directories
40# ---------------------------------------------------------------------------
42try:
43 _PACKAGE_DIR = importlib_resources.files("local_deep_research") / "web"
44 with importlib_resources.as_file(_PACKAGE_DIR) as _pkg:
45 STATIC_DIR = (_pkg / "static").as_posix()
46 TEMPLATE_DIR = (_pkg / "templates").as_posix()
47except Exception:
48 STATIC_DIR = str(Path("static").resolve())
49 TEMPLATE_DIR = str(Path("templates").resolve())
51# Templates shared across all routers (defined in template_config to avoid circular imports)
52from .template_config import (
53 templates,
54)
56# ---------------------------------------------------------------------------
57# Branded error pages (404 / 413)
58# ---------------------------------------------------------------------------
60#: Title/message copy for each status code the branded error page covers.
61#: Ports the copy from PR #5424 (Flask's ``app_factory.py::
62#: register_error_handlers``, retired by the FastAPI migration in #3299).
63_BRANDED_ERROR_COPY: dict[int, tuple[str, str]] = {
64 404: (
65 "Page not found",
66 "The page you're looking for doesn't exist or may have moved.",
67 ),
68 413: (
69 "Request too large",
70 "The file or request you sent is larger than the allowed limit.",
71 ),
72 # Only ``/`` renders this one (``_database_unavailable_page``): the
73 # retryable 503 of ``database_unavailable_exception``.
74 503: (
75 "Temporarily unavailable",
76 "Your database is temporarily unavailable. "
77 "Please try again in a moment.",
78 ),
79}
82def _render_branded_error_page(
83 request: Request,
84 status_code: int,
85 *,
86 title: str | None = None,
87 message: str | None = None,
88):
89 """Render ``templates/pages/error.html`` for a browser-facing error.
91 Ports the defensive design of the Flask handler this replaces (PR
92 #5424): rendering depends on session data, CSRF token generation, and
93 Jinja globals (``url_for``, the frontend-constants injection in
94 ``template_config.py``'s ``_LDRTemplates.TemplateResponse``) that are
95 not guaranteed to be populated in every environment this can run in —
96 most notably, a minimal test harness may not populate ``scope["app"]``
97 at all (no ``url_for``), or may not install ``SessionMiddleware`` at
98 all (no ``scope["session"]``). A template that itself raises must
99 never turn a 404/413 into an unhandled 500, so any failure here is
100 swallowed and logged at debug; callers fall back to the previous
101 plain-text/JSON response.
103 ``title`` and ``message`` replace the copy for ``status_code``. The
104 template escapes both.
106 Returns the rendered ``Response`` on success, or ``None`` on any
107 failure.
108 """
109 default_title, default_message = _BRANDED_ERROR_COPY.get(
110 status_code, ("Something went wrong", "Please try again.")
111 )
112 title = default_title if title is None else title
113 message = default_message if message is None else message
114 from ..security.egress.policy import DEFAULT_EGRESS_SCOPE
116 # If a session is present -- real or a caller-supplied placeholder
117 # dict (see BodySizeLimitMiddleware._send_413) -- render against a
118 # *copy* of the scope with a snapshotted "session", never the live
119 # SessionMiddleware ``Session`` object. `_LDRTemplates.
120 # TemplateResponse` (template_config.py:41) does
121 # `getattr(request, "session", None)` unconditionally, and
122 # Starlette's `Request.session` property calls
123 # `session.mark_accessed()` purely by being *read* -- which alone
124 # makes SessionMiddleware add `Vary: Cookie` to the response, before
125 # any of the "not in context" guards below even run. A plain dict
126 # has no `mark_accessed`, so touching the copy here can never mark
127 # the real session accessed or modified, which is what keeps this
128 # from ever adding Set-Cookie/Vary either.
129 #
130 # If there is no session at all (no key in scope -- SessionMiddleware
131 # never ran on this scope, e.g. a minimal test harness, or a caller
132 # that has not populated one), leave `request` untouched: the
133 # `getattr` above then raises the same ``AssertionError`` it always
134 # did, caught by this function's own `except` below, and rendering
135 # falls back to the plain-text/JSON response exactly as before this
136 # session-copy was introduced -- there is no live session object to
137 # protect in that case, so there is nothing to fix.
138 real_session = request.scope.get("session")
139 if hasattr(real_session, "get"):
140 scope = dict(request.scope)
141 scope["session"] = dict(real_session)
142 render_request = Request(scope)
143 else:
144 render_request = request
146 try:
147 return templates.TemplateResponse(
148 request=render_request,
149 name="pages/error.html",
150 context={
151 "request": render_request,
152 "status_code": status_code,
153 "error_title": title,
154 "error_message": message,
155 # error.html is a standalone page (does not extend
156 # base.html) and never reads any of these four keys, so
157 # any value satisfies the template — but *_LDRTemplates.
158 # TemplateResponse* treats a missing key as "go compute
159 # the real one": CSRF-token generation (:51, mutates the
160 # session copy above -- harmless, but would still cost a
161 # `secrets.token_hex` call), a flash-message pop (:58,
162 # would drain the *copy*, which is why the real pending
163 # flashes still reach the next real page), and — worst —
164 # an egress-scope DB lookup for any logged-in user (:145)
165 # on every 404/413, including ones DatabaseMiddleware
166 # never opens a session for (the session copy still has
167 # ``username``, so only supplying this explicitly avoids
168 # the DB round trip). Supplying all four here
169 # short-circuits every "not in context" guard so none of
170 # that work ever runs for an error page.
171 "csrf_token": "",
172 "get_flashed_messages": lambda *_a, **_k: [],
173 "egress_scope": DEFAULT_EGRESS_SCOPE,
174 "has_encryption": False,
175 },
176 status_code=status_code,
177 )
178 except Exception:
179 logger.opt(exception=True).debug(
180 "Failed to render branded error page for status {}; falling "
181 "back to a plain response",
182 status_code,
183 )
184 return None
187def _database_unavailable_page(request: Request, exc: HTTPException):
188 """The branded page for ``/``'s retryable 503, or None for the JSON one.
190 ``exc`` is ``database_unavailable_exception()``: the user's database
191 cannot be checked or opened right now (a rekey sidecar fault such as
192 an fd-exhaustion spike). ``/`` is where a browser lands after login,
193 and ``/auth/login`` sends every signed-in cookie back to it, so a fault
194 that lasts would otherwise leave the browser on a raw JSON body. The
195 branded page keeps the status and ``Retry-After``, and its button
196 retries ``/``. It renders without a database read and from a copy of
197 the session (see ``_render_branded_error_page``), so the session stays
198 exactly as the 503 promises. Returns None, so that the caller raises
199 ``exc`` for the usual JSON answer, for a caller that asked for JSON or
200 when the page cannot be rendered.
201 """
202 if _is_api_request(request):
203 return None
204 page = _render_branded_error_page(request, exc.status_code)
205 if page is None: 205 ↛ 206line 205 didn't jump to line 206 because the condition on line 205 was never true
206 return None
207 for name, value in (exc.headers or {}).items():
208 page.headers[name] = value
209 return page
212# ---------------------------------------------------------------------------
213# Secret key (same logic as Flask app_factory)
214# ---------------------------------------------------------------------------
217def _load_secret_key() -> str:
218 """Load or generate a persistent SECRET_KEY for session signing.
220 Falling back to an in-memory key on read errors silently invalidates
221 every existing session on restart and produces no operator alert,
222 so this raises a hard error instead.
223 """
224 from ..config.paths import get_data_directory
226 secret_key_file = Path(get_data_directory()) / ".secret_key"
228 from ..security.directory_creation import create_directory
230 create_directory(
231 secret_key_file.parent, context="secret key file directory"
232 )
233 new_key = secrets.token_hex(32)
234 try:
235 fd = os.open(
236 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Fixed .secret_key below the operator data root; creation is exclusive with mode 0600
237 str(secret_key_file),
238 os.O_WRONLY | os.O_CREAT | os.O_EXCL,
239 0o600,
240 )
241 try:
242 os.write(fd, new_key.encode())
243 finally:
244 os.close(fd)
245 logger.info("Generated new SECRET_KEY for this installation")
246 return new_key
247 except FileExistsError:
248 # File exists — read it. Any error here is fatal: an ephemeral
249 # in-memory key would silently invalidate every existing session
250 # on restart and prevent any future restart from being consistent.
251 try:
252 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Fixed .secret_key below the operator data root; requests cannot select this filename
253 with open(secret_key_file, "r", encoding="utf-8") as f:
254 key = f.read().strip()
255 except Exception as e:
256 raise RuntimeError(
257 f"Cannot read SECRET_KEY file at {secret_key_file}: {e}. "
258 "Fix the file permissions/contents or remove the file to "
259 "regenerate (this will invalidate all sessions)."
260 ) from e
261 # Empty / whitespace-only file is also fatal — itsdangerous would
262 # accept it and derive a deterministic signing key from no secret,
263 # silently breaking session integrity.
264 if not key:
265 raise RuntimeError(
266 f"SECRET_KEY file at {secret_key_file} is empty. "
267 "Delete the file to regenerate a new key (this will "
268 "invalidate all existing sessions)."
269 )
270 return key
271 except OSError as e:
272 # Cannot write the key file. Same reasoning — refuse rather than
273 # silently fall back to a non-persistent key.
274 raise RuntimeError(
275 f"Cannot write SECRET_KEY file at {secret_key_file}: {e}. "
276 "Fix the data directory permissions and retry."
277 ) from e
280SECRET_KEY = _load_secret_key()
283def warn_if_threadpool_exceeds_db_pool(worker_threads: int) -> bool:
284 """Warn when the AnyIO worker pool is larger than a user's DB pool.
286 Raising ``web.threadpool_max_threads`` above the per-user pool capacity is
287 not merely a tuning choice. ``WorkerCleanupAPIRoute`` returns each sync
288 route's connections when that request finishes, but every concurrently
289 active DB-touching worker can still hold one pool slot. More workers than
290 slots therefore permit a burst from one user to exhaust the pool before
291 any request reaches its cleanup boundary.
293 Below the pool capacity that is harmless, which is why this warns instead
294 of failing. Above it, one user's concurrent requests can exhaust their own
295 pool, after which every further request waits out ``pool_timeout``.
297 Returns True when the warning fired, so it can be tested directly rather
298 than only through a lifespan startup.
299 """
300 from ..database.pool_config import MAX_OVERFLOW, POOL_SIZE
302 pool_capacity = POOL_SIZE + MAX_OVERFLOW
303 if worker_threads <= pool_capacity:
304 return False
305 logger.warning(
306 "web.threadpool_max_threads ({}) exceeds the per-user database pool "
307 "capacity ({} = pool_size {} + max_overflow {}). Concurrent "
308 "synchronous routes can each hold one connection, so a single user's "
309 "concurrent requests can exhaust their pool and then block for "
310 "pool_timeout. Keep this at or below {}.",
311 worker_threads,
312 pool_capacity,
313 POOL_SIZE,
314 MAX_OVERFLOW,
315 pool_capacity,
316 )
317 return True
320# ---------------------------------------------------------------------------
321# Lifespan: startup / shutdown
322# ---------------------------------------------------------------------------
325@asynccontextmanager
326async def lifespan(app: FastAPI):
327 """Manage application lifecycle — start/stop background services."""
328 # --- Startup ---
329 logger.info(f"Starting Local Deep Research v{__version__}")
331 # Capture uvicorn's running event loop so background threads can
332 # dispatch Socket.IO emits via run_coroutine_threadsafe. Without
333 # this, emits from research workers / log queue silently no-op
334 # (asyncio.get_event_loop() in a worker thread doesn't return uvicorn's loop).
335 import asyncio as _asyncio
336 from .services.socketio_asgi import (
337 init_lock,
338 log_socketio_cors_policy,
339 set_main_loop,
340 )
342 set_main_loop(_asyncio.get_running_loop())
343 # Eagerly create the Socket.IO subscription lock now that the loop is
344 # running, so the first connect/subscribe events don't race on lazy init.
345 init_lock()
346 # Report the WebSocket origin policy here rather than at socketio_asgi
347 # import time: that import happens while package logging is still
348 # disabled (before config_logger() runs), which silently dropped the
349 # allow-all INFO/WARNING. Logging-only, so never let it block startup.
350 try:
351 log_socketio_cors_policy()
352 except Exception:
353 logger.exception("Failed to log the Socket.IO CORS policy")
355 # Size the AnyIO worker pool that serves every plain `def` route.
356 #
357 # Starlette runs sync handlers via `anyio.to_thread.run_sync`, whose
358 # default CapacityLimiter is 40 threads — and that pool is SHARED with
359 # async dependency solving and response validation, so exhausting it
360 # degrades async routes too. This app has ~248 sync routes against 65
361 # async ones, each sync request holding a worker for its full duration
362 # (including a first-call SQLCipher open), and it runs with workers=1,
363 # so there is no second process to absorb the overflow. Measured
364 # symptom: /api/v1/health went 0.8ms -> 9.8s at 80 concurrent requests,
365 # past the 8s Docker healthcheck timeout — i.e. the container reports
366 # unhealthy under load rather than merely slow.
367 #
368 # Left at AnyIO's default unless an operator opts in, so this changes
369 # nothing by default: raising it is not free (context-switch overhead,
370 # and more threads contend for the same per-user QueuePool), and the
371 # right value depends on the deployment's core count and workload. The
372 # point is to make an invisible framework default into a deliberate,
373 # tunable decision — the migration hazard here is not the number, it is
374 # that nobody knows the number exists.
375 try:
376 from ..settings.env_registry import (
377 get_env_setting as _get_env_setting,
378 )
380 configured = _get_env_setting("web.threadpool_max_threads")
381 if configured: 381 ↛ 382line 381 didn't jump to line 382 because the condition on line 381 was never true
382 import anyio.to_thread
384 limiter = anyio.to_thread.current_default_thread_limiter()
385 previous = limiter.total_tokens
386 limiter.total_tokens = int(configured)
387 logger.info(
388 "AnyIO worker pool resized: {} -> {} threads "
389 "(LDR_WEB_THREADPOOL_MAX_THREADS)",
390 previous,
391 limiter.total_tokens,
392 )
394 warn_if_threadpool_exceeds_db_pool(limiter.total_tokens)
395 except Exception:
396 # Never let a tuning knob stop the server from starting.
397 logger.exception(
398 "Could not apply web.threadpool_max_threads; "
399 "continuing with the AnyIO default"
400 )
402 # Route stdlib loggers through loguru
403 for name in ("uvicorn", "uvicorn.access", "uvicorn.error"):
404 uv_logger = logging.getLogger(name)
405 uv_logger.setLevel(logging.WARNING)
406 if not any(isinstance(h, InterceptHandler) for h in uv_logger.handlers):
407 uv_logger.addHandler(InterceptHandler())
409 ap_logger = logging.getLogger("apscheduler")
410 ap_logger.setLevel(logging.WARNING)
411 if not any(isinstance(h, InterceptHandler) for h in ap_logger.handlers):
412 ap_logger.addHandler(InterceptHandler())
414 # Log data location
415 from ..config.paths import get_data_directory
416 from ..database.encrypted_db import db_manager
418 data_dir = get_data_directory()
419 logger.info("=" * 60)
420 logger.info("DATA STORAGE INFORMATION")
421 logger.info("=" * 60)
422 logger.info(f"Data directory: {data_dir}")
423 logger.info(
424 "Databases: Per-user encrypted databases in encrypted_databases/"
425 )
426 if db_manager.has_encryption: 426 ↛ 429line 426 didn't jump to line 429 because the condition on line 426 was always true
427 logger.info("SECURITY: Databases are encrypted with SQLCipher.")
428 else:
429 logger.warning(
430 "SECURITY NOTICE: SQLCipher is not available - databases are NOT encrypted."
431 )
432 logger.info("=" * 60)
434 # Restrict research_outputs/ (optional plaintext report backups) to this
435 # OS account, so a directory left at 0755 by an older release becomes
436 # 0700 whether or not any user has report file backups enabled. The
437 # function logs its own failure and remedy; FileReportStorage refuses a
438 # directory that is not private, so a failure here only disables file
439 # backups. Wrapped so it can never block server boot.
440 try:
441 from ..config.paths import restrict_research_outputs_directory
443 restrict_research_outputs_directory()
444 except Exception:
445 logger.exception("Research outputs directory startup check failed")
447 # Surface a cipher misconfiguration that otherwise only shows up as
448 # affected users getting "Invalid username or password": a relaxed
449 # SQLCipher KDF (test mode) on a deployment that already holds real user
450 # databases. No-op on fresh installs and when the effective KDF is at the
451 # production floor. Wrapped so a check failure can never block server boot.
452 try:
453 from ..database.sqlcipher_utils import (
454 warn_if_weak_kdf_with_existing_databases,
455 )
457 if db_manager.has_encryption: 457 ↛ 463line 457 didn't jump to line 463 because the condition on line 457 was always true
458 warn_if_weak_kdf_with_existing_databases(db_manager.data_dir)
459 except Exception:
460 logger.exception("Weak-KDF startup configuration check failed")
462 # Initialize Theme helper
463 from .themes import theme_registry
465 try:
466 static_dir = Path(STATIC_DIR)
467 themes_css_path = static_dir / "css" / "themes.css"
468 combined_css = theme_registry.get_combined_css()
469 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Fixed css/themes.css below the installed package static directory at startup
470 themes_css_path.write_text(combined_css, encoding="utf-8")
471 logger.debug(
472 f"Generated themes.css with {len(theme_registry.themes)} themes"
473 )
474 except Exception:
475 logger.warning("Error generating combined themes.css")
477 # Start log queue processor (drains background-thread DB log entries).
478 #
479 # Wrapped like every other optional startup step in this lifespan (the
480 # AnyIO threadpool knob, the weak-KDF check, themes.css). The lifespan is
481 # two-tier and an UNGUARDED step that raises produces
482 # `lifespan.startup.failed`, so uvicorn never serves at all -- pinned by
483 # tests/web/test_lifespan_startup_shutdown.py.
484 # `start_log_queue_processor` ends in `threading.Thread(...).start()`,
485 # which raises `RuntimeError: can't start new thread` under a container
486 # pids / RLIMIT_NPROC ceiling; a best-effort logging daemon must never be
487 # the reason the server fails to boot. main guards the same call in
488 # `web/app.py::main()` (PR #3488) and the guard was lost in the port.
489 # The matching `stop_log_queue_processor()` on the shutdown path is
490 # itself wrapped and is a no-op when no daemon ever started, so it stays
491 # reachable either way; logging degrades to the shutdown flush.
492 from ..utilities.log_utils import start_log_queue_processor
494 try:
495 start_log_queue_processor()
496 except Exception:
497 logger.exception(
498 "Failed to start log queue processor; continuing without the "
499 "background log drain (queued entries flush at shutdown instead)"
500 )
502 # Start research queue processor — enabled by default in normal app runs,
503 # but default-off under pytest so app fixtures don't each spawn a daemon
504 # thread. An explicit web.queue_processor.enabled env override still wins
505 # (tests that need the processor opt in). Ports #5055 from the retired
506 # Flask app_factory to the FastAPI lifespan. The singleton is imported
507 # unconditionally (cheap — only .start() spawns the thread) so the
508 # shutdown path below can always call .stop().
509 from ..settings.env_registry import get_env_setting, registry
510 from .queue.processor_v2 import queue_processor
512 queue_processor_enabled = get_env_setting(
513 "web.queue_processor.enabled", True
514 )
515 _qp_setting = registry.get_setting_object("web.queue_processor.enabled")
516 _qp_env_set = bool(_qp_setting and _qp_setting.is_set)
517 if os.getenv("PYTEST_CURRENT_TEST") and not _qp_env_set: 517 ↛ 520line 517 didn't jump to line 520 because the condition on line 517 was always true
518 queue_processor_enabled = False
520 if queue_processor_enabled: 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true
521 queue_processor.start()
522 logger.info("Started research queue processor v2")
523 else:
524 # Disabling the processor also disables the ONLY drain path for
525 # ``pending_operations``: research worker threads that cannot reach
526 # the DB directly (e.g. password lookup failed) fall back to
527 # ``queue_progress_update`` / ``queue_error_update``, and the only
528 # consumer of that queue is ``_drain_pending_operations``, which runs
529 # inside the loop started just above. Under Flask a second, always-on
530 # ``before_request`` hook drained it too; the FastAPI port has no
531 # equivalent. Directly-dispatched research still runs while the
532 # processor is off, so a terminal FAILED status raised on that
533 # fallback path would be queued and then silently dropped by TTL
534 # eviction. Warn rather than start a thread the operator turned off.
535 logger.warning(
536 "Queue processor v2 disabled — not starting. Queued researches "
537 "will not be dispatched, and progress/terminal-status updates "
538 "that fall back to the pending-operations queue will not be "
539 "persisted (they expire via TTL)."
540 )
542 # Start news scheduler
543 news_scheduler = None
544 try:
545 from ..settings.env_registry import get_env_setting
547 scheduler_enabled = get_env_setting("news.scheduler.enabled", True)
548 if scheduler_enabled: 548 ↛ 549line 548 didn't jump to line 549 because the condition on line 548 was never true
549 from ..scheduler.background import get_background_job_scheduler
550 from ..settings.manager import SettingsManager
552 # Startup-only: lifespan runs once before the server accepts
553 # traffic, so blocking here cannot stall in-flight requests.
554 settings_manager = SettingsManager() # allow: sync-in-async
555 news_scheduler = get_background_job_scheduler()
556 news_scheduler.initialize_with_settings(settings_manager)
557 news_scheduler.start()
558 logger.info("News scheduler started")
559 except Exception:
560 logger.exception("Failed to initialize news scheduler")
562 # Start connection cleanup scheduler
563 cleanup_scheduler = None
564 try:
565 from .auth.connection_cleanup import start_connection_cleanup_scheduler
566 from .auth.session_manager import session_manager
568 cleanup_scheduler = start_connection_cleanup_scheduler(
569 session_manager, db_manager
570 )
571 except Exception:
572 logger.warning("Failed to start cleanup scheduler")
574 # Uvicorn drains requests before entering lifespan shutdown. Its
575 # timeout_graceful_shutdown (10s) drain timeout cancels request tasks,
576 # then still sends the lifespan shutdown event; it does not cancel
577 # this generator at the yield.
578 # Compose allows 60s for both phases. Scheduler shutdown can still wait
579 # for an in-flight job, so this is an outer grace period, not a bound on
580 # each cleanup step. SIGKILL or cancellation of the lifespan itself can
581 # skip the code below. An atexit handler cannot recover from SIGKILL and
582 # would double-close databases on normal exit.
584 yield # --- App is running ---
586 # --- Shutdown ---
587 logger.info("Shutting down Local Deep Research...")
589 if news_scheduler: 589 ↛ 590line 589 didn't jump to line 590 because the condition on line 589 was never true
590 try:
591 news_scheduler.stop()
592 logger.info("News scheduler stopped")
593 except Exception:
594 logger.exception("Error stopping news scheduler")
596 if cleanup_scheduler: 596 ↛ 606line 596 didn't jump to line 606 because the condition on line 596 was always true
597 try:
598 # wait=True so in-flight cleanup jobs finish before we close
599 # the DB below — otherwise they hit a closed pool.
600 cleanup_scheduler.shutdown(wait=True)
601 except Exception:
602 logger.debug("Error stopping cleanup scheduler")
604 # Stop the research queue processor so a uvicorn reload doesn't leave
605 # an old daemon thread holding locks while the new process starts.
606 try:
607 queue_processor.stop()
608 except Exception:
609 logger.debug("Error stopping queue processor")
611 try:
612 from ..embeddings.model_cache import clear_sentence_transformer_cache
614 # This lock attempt never waits for inference. Queueing it on the
615 # default executor would still wait when inference occupies that pool,
616 # delaying log/database cleanup. Traffic has drained at this point.
617 clear_sentence_transformer_cache(wait=False)
618 except Exception:
619 logger.debug("Error clearing embedding model cache during shutdown")
621 # Flush pending DB log writes BEFORE closing databases; `database_sink`
622 # swallows errors silently, so anything flushed after close is just
623 # dropped.
624 from ..utilities.log_utils import (
625 flush_log_queue,
626 stop_log_queue_processor,
627 )
629 try:
630 flush_log_queue()
631 stop_log_queue_processor()
632 except Exception:
633 logger.debug("Error flushing log queue during shutdown")
635 try:
636 db_manager.close_all_databases()
637 logger.info("Database connections closed")
638 except Exception:
639 logger.exception("Error closing databases")
642# ---------------------------------------------------------------------------
643# ASGI Middleware (pure ASGI — NOT BaseHTTPMiddleware)
644# ---------------------------------------------------------------------------
647def _is_private_ip(ip_str: str) -> bool:
648 """Check if IP is a private/local network address.
650 Returns True for private IPs (RFC 1918), localhost, and non-parseable
651 strings (e.g. 'testclient' from TestClient). Non-parseable strings
652 are treated as private to avoid adding Secure flag in test/dev contexts.
654 A 6to4, Teredo or NAT64 peer is NOT private, whatever ``ipaddress``
655 says: such an address reaches this server through a relay or
656 translator from somewhere else entirely. Both consumers depend on
657 that -- the forwarded-proto advice must not be plantable by a remote
658 client arriving over a tunnel, and the "serving HTTP to a public
659 client" warning must not stay silent for one. The ranges are the
660 shared ``network_utils.IPV6_TRANSITION_NETWORKS``, the same ones the
661 rate limiter's proxy-trust decision excludes.
663 An IPv4-mapped address (``::ffff:a.b.c.d``, how an IPv4 client appears
664 on a dual-stack listener) is judged by the IPv4 address it wraps,
665 unwrapped here as ``network_utils.is_private_peer_ip`` does. CPython
666 3.12.0 to 3.12.3 answer from their IPv6 table instead, which lists
667 ``::ffff:0:0/96`` as private, so a public IPv4 client would read as
668 private on those versions.
669 """
670 if not ip_str:
671 return True
672 try:
673 ip = ipaddress.ip_address(ip_str)
674 except ValueError:
675 return True # Non-IP strings (e.g. "testclient") treated as private
676 if ip.version == 6 and ip.ipv4_mapped is not None:
677 ip = ip.ipv4_mapped
678 if is_ipv6_transition_address(ip):
679 return False
680 return ip.is_private or ip.is_loopback
683class SecurityHeadersMiddleware:
684 """Pure ASGI middleware for security headers.
686 Ports Flask's SecurityHeaders + ServerHeaderMiddleware to ASGI.
687 """
689 # Pre-compute static header values
690 CSP = (
691 "default-src 'self'; "
692 "connect-src 'self'; "
693 "script-src 'self' 'unsafe-inline'; "
694 "style-src 'self' 'unsafe-inline'; "
695 "font-src 'self' data:; "
696 "img-src 'self' data:; "
697 "media-src 'self'; "
698 "worker-src blob:; "
699 "child-src 'self' blob:; "
700 "frame-src 'self'; "
701 "frame-ancestors 'self'; "
702 "manifest-src 'self'; "
703 "object-src 'none'; "
704 "base-uri 'self'; "
705 "form-action 'self';"
706 )
707 PERMISSIONS = (
708 "geolocation=(), midi=(), camera=(), usb=(), "
709 "magnetometer=(), accelerometer=(), gyroscope=(), "
710 "microphone=(), payment=(), sync-xhr=(), document-domain=()"
711 )
713 @classmethod
714 def unconditional_headers(cls) -> list[tuple[bytes, bytes]]:
715 """The security headers stamped on every HTTP response.
717 Exposed separately because this middleware cannot cover every
718 response. Starlette's ``ServerErrorMiddleware`` sits OUTSIDE every
719 middleware added via ``app.add_middleware`` — it is installed by
720 ``build_middleware_stack`` itself — and when an exception handler
721 is registered for the bare ``Exception`` class, Starlette wires it
722 in as that middleware's handler. Its response is written to the raw
723 ASGI ``send``, bypassing this middleware entirely, so a 500 from an
724 unregistered exception type would otherwise carry NO security
725 headers at all. The catch-all handler reuses this list to stamp
726 them itself; keeping one source stops the two copies drifting.
728 Excludes the two conditional headers (HSTS, which needs a secure
729 scheme, and cache-control, which is skipped for static asset routes).
730 """
731 return [
732 (b"content-security-policy", cls.CSP.encode()),
733 (b"x-frame-options", b"SAMEORIGIN"),
734 (b"x-content-type-options", b"nosniff"),
735 (b"cross-origin-opener-policy", b"same-origin"),
736 (b"cross-origin-embedder-policy", b"credentialless"),
737 (b"cross-origin-resource-policy", b"same-origin"),
738 (b"permissions-policy", cls.PERMISSIONS.encode()),
739 (b"referrer-policy", b"strict-origin-when-cross-origin"),
740 ]
742 @classmethod
743 def cache_headers(cls) -> list[tuple[bytes, bytes]]:
744 """No-store cache headers stamped on responses outside static routes.
746 Split out from ``unconditional_headers()`` (Cache-Control/Pragma/
747 Expires are conditional on path, not truly unconditional) so the
748 catch-all 500 handler below can reuse the exact same values
749 instead of hand-copying them — main's Flask ``after_request``
750 (``security/security_headers.py``) set all three:
751 ``Cache-Control``, ``Pragma`` (HTTP/1.0 compatibility), and
752 ``Expires: 0``. The ASGI port originally dropped ``Expires``;
753 restored here for parity. Impact of the omission was low
754 (``Cache-Control: no-store`` already dominates for any HTTP/1.1
755 cache) but it's a free one-line fix once the two are split out.
756 """
757 return [
758 (
759 b"cache-control",
760 b"no-store, no-cache, must-revalidate, max-age=0",
761 ),
762 (b"pragma", b"no-cache"),
763 (b"expires", b"0"),
764 ]
766 def __init__(self, app):
767 self.app = app
769 @classmethod
770 def response_headers(
771 cls, *, path: str, scheme: str, status: int
772 ) -> list[tuple[bytes, bytes]]:
773 """Apply the same header policy inside and outside the middleware."""
774 headers = cls.unconditional_headers()
775 if scheme == "https":
776 headers.append(
777 (
778 b"strict-transport-security",
779 b"max-age=31536000; includeSubDomains",
780 )
781 )
782 legacy_asset = path.startswith("/redirect-static/") and status in (
783 200,
784 206,
785 304,
786 )
787 if not path.startswith("/static/") and not legacy_asset:
788 headers.extend(cls.cache_headers())
789 return headers
791 async def __call__(self, scope, receive, send):
792 if scope["type"] != "http":
793 await self.app(scope, receive, send)
794 return
796 path = scope.get("path", "")
798 async def send_wrapper(message):
799 if message["type"] == "http.response.start":
800 additions = self.response_headers(
801 path=path,
802 scheme=scope.get("scheme", "http"),
803 status=message["status"],
804 )
805 # Keep the route's streaming directive while replacing its
806 # caching policy. SSE must remain untransformed by proxies,
807 # and dynamic responses must still be no-store.
808 if any(
809 name.lower() == b"cache-control"
810 and any(
811 directive.strip().lower() == b"no-transform"
812 for directive in value.split(b",")
813 )
814 for name, value in message.get("headers", [])
815 ):
816 additions = [
817 (
818 name,
819 value + b", no-transform"
820 if name == b"cache-control"
821 else value,
822 )
823 for name, value in additions
824 ]
825 replaced = {name for name, _ in additions} | {b"server"}
826 headers = [
827 (name, value)
828 for name, value in message.get("headers", [])
829 if name.lower() not in replaced
830 ]
831 headers.extend(additions)
832 message = {**message, "headers": headers}
834 await send(message)
836 await self.app(scope, receive, send_wrapper)
839class _RequestBodyTooLarge(Exception):
840 """Raised by BodySizeLimitMiddleware's receive wrapper on overflow."""
843#: Separate, much smaller cap applied to non-multipart bodies (JSON and
844#: everything else that isn't a file upload) by ``BodySizeLimitMiddleware``.
845#:
846#: ``await request.json()`` -> ``json.loads`` is synchronous and runs on
847#: the single uvicorn event loop this app is served with (workers=1), so
848#: one caller's oversized body stalls every concurrent user. Measured on
849#: this branch: a 104 MB JSON body stalled the loop for 637 ms, during
850#: which 12 concurrent GETs spiked from ~3 ms to 184 ms latency. The only
851#: prior bound was the general ``max_body_size`` cap below
852#: (MAX_FILES_PER_REQUEST(200) x MAX_FILE_SIZE(3GB) ~= 600 GB) — sized for
853#: multipart file uploads, not a practical limit for a single JSON body.
854#:
855#: 100 MB matches the existing precedent in
856#: ``web/routers/notes.py::_MAX_JSON_BODY_BYTES``
857#: (``2 * NOTE_CONTENT_MAX_BYTES``, i.e. 2x the 50 MB note-content cap),
858#: which already accepts this order-of-magnitude stall for its own
859#: pre-parse JSON body gate. No other JSON endpoint in the app needs
860#: anywhere near this: research/chat/benchmark/settings JSON bodies are
861#: small query/config/message payloads (KB, not MB) — see PR review notes
862#: for the survey of ``await request.json()`` call sites. The two routes
863#: that genuinely need multi-hundred-MB bodies (``/api/upload/pdf`` and
864#: ``/library/api/collections/{id}/upload``) are multipart file uploads,
865#: not JSON, and keep the large ``max_body_size`` cap below.
866_DEFAULT_MAX_JSON_BODY_SIZE = 100 * 1024 * 1024 # 100 MB
868#: The cap every OTHER JSON route gets. 100 MB above is what the notes
869#: route needs, but it is a weak mitigation for the defect this cap exists
870#: to address: ``json.loads`` runs synchronously on the single uvicorn
871#: event loop (workers=1), so body size translates directly into a stall
872#: that freezes EVERY concurrent user, not just the caller. Measured on
873#: this branch: 104 MB -> 498-637 ms, 34 MB -> ~128 ms, 8 MB -> ~38 ms.
874#: A 100 MB cap therefore still permits a ~600 ms full-service stall,
875#: repeatable at will by any authenticated caller.
876#:
877#: 16 MB holds the worst case to roughly 60 ms while leaving ~4x headroom
878#: over the largest realistic legitimate body found in a survey of every
879#: ``await request.json()`` call site (~4 MB). Routes that genuinely need
880#: more are listed in _LARGE_JSON_BODY_PREFIXES and keep the 100 MB cap.
881_DEFAULT_MAX_SMALL_JSON_BODY_SIZE = 16 * 1024 * 1024 # 16 MB
883#: Path prefixes allowed the full ``_DEFAULT_MAX_JSON_BODY_SIZE``. Keep
884#: this list as short as possible: every entry is a route that can stall
885#: the event loop for hundreds of ms. ``notes`` earns it because
886#: NOTE_CONTENT_MAX_BYTES is 50 MB and its handler already does its own
887#: bounded pre-parse read (notes.py::_notes_json_body).
888_LARGE_JSON_BODY_PREFIXES = ("/notes/",)
890#: The only two routes that genuinely consume a multipart upload body
891#: (both do ``await request.form()`` and pull ``UploadFile``s out of it).
892#: The large ``max_body_size`` cap is granted on PATH, never on the
893#: client's declared Content-Type. Starlette's ``Request.json()`` is
894#: ``json.loads(await self.body())`` -- it never inspects Content-Type --
895#: so a body labelled ``multipart/form-data`` is still parsed as JSON by
896#: any route that asks for JSON. Choosing the cap from the header alone
897#: therefore handed every one of the app's ``request.json()`` routes the
898#: ~600 GB upload cap (200 files x 3 GB) for the cost of one mislabelled
899#: header, while the route went on parsing the body as JSON.
900_MULTIPART_UPLOAD_PATHS = frozenset({"/api/upload/pdf"})
901_MULTIPART_UPLOAD_RE = re.compile(r"^/library/api/collections/[^/]+/upload$")
904def _is_multipart_upload_path(path: str) -> bool:
905 """True for the two routes allowed to carry an upload-sized body."""
906 return path in _MULTIPART_UPLOAD_PATHS or bool(
907 _MULTIPART_UPLOAD_RE.match(path)
908 )
911class BodySizeLimitMiddleware:
912 """Pure ASGI middleware enforcing a global request-body size cap.
914 Ports Flask's ``MAX_CONTENT_LENGTH`` (app_factory set it to
915 ``MAX_FILES_PER_REQUEST * MAX_FILE_SIZE``; Werkzeug rejected any
916 larger body with 413) plus the ``@app.errorhandler(413)`` content
917 negotiation. Without it every body-reading code path — including the
918 CSRF middleware, which buffers form bodies to extract the token —
919 would buffer arbitrarily large request bodies into memory.
921 The declared Content-Length is checked before the app runs; bodies
922 streamed without one (chunked transfer) are counted chunk by chunk so
923 the cap cannot be bypassed.
925 On top of that global cap, non-multipart requests (JSON bodies, but
926 also anything with a missing/spoofed Content-Type — a route can call
927 ``await request.json()`` regardless of what Content-Type the client
928 sent, so gating strictly on ``application/json`` would leave that as
929 a bypass) are additionally capped at the much smaller
930 ``max_json_body_size``. Multipart requests (the two real upload
931 routes) are exempt from that smaller cap and still enforce only
932 ``max_body_size`` — see ``_DEFAULT_MAX_JSON_BODY_SIZE`` above for the
933 justification and measurements.
934 """
936 def __init__(
937 self,
938 app,
939 max_body_size: int | None = None,
940 max_json_body_size: int | None = None,
941 max_large_json_body_size: int | None = None,
942 ):
943 self.app = app
944 if max_body_size is None:
945 from ..security.file_upload_validator import FileUploadValidator
947 max_body_size = (
948 FileUploadValidator.MAX_FILES_PER_REQUEST
949 * FileUploadValidator.MAX_FILE_SIZE
950 )
951 self.max_body_size = max_body_size
952 # `max_json_body_size` is the cap for ORDINARY JSON routes, i.e.
953 # almost everything. `max_large_json_body_size` applies only to the
954 # few prefixes in _LARGE_JSON_BODY_PREFIXES that legitimately carry
955 # a big body. Keeping the ordinary case as the injectable default
956 # means a caller passing a small cap gets it applied where it
957 # matters, rather than silently only affecting notes.
958 if max_json_body_size is None:
959 max_json_body_size = _DEFAULT_MAX_SMALL_JSON_BODY_SIZE
960 self.max_json_body_size = max_json_body_size
961 if max_large_json_body_size is None:
962 max_large_json_body_size = _DEFAULT_MAX_JSON_BODY_SIZE
963 self.max_large_json_body_size = max(
964 max_large_json_body_size, max_json_body_size
965 )
967 async def _send_413(self, scope, receive, send):
968 path = scope.get("path", "")
969 # Same negotiation as main's 413 errorhandler (_is_api_path).
970 if "/api/" in path or path.endswith("/api"):
971 body = b'{"error": "Request too large"}'
972 content_type = b"application/json"
973 await send(
974 {
975 "type": "http.response.start",
976 "status": 413,
977 "headers": [
978 (b"content-type", content_type),
979 (b"content-length", str(len(body)).encode()),
980 ],
981 }
982 )
983 await send({"type": "http.response.body", "body": body})
984 return
986 # Only the declared-length rejection (the fast path in __call__,
987 # before `self.app` — and therefore SessionMiddleware — ever
988 # runs) reaches this with no "session" key in `scope` at all;
989 # `setdefault` supplies an empty placeholder there, without
990 # adding keys to the original scope, so the branded page still
991 # renders instead of falling back to plain text for that path.
992 # On the streamed (chunked) path, this runs from `__call__`'s
993 # `except _RequestBodyTooLarge` handler AFTER `self.app(...)`
994 # was entered, so SessionMiddleware has already populated
995 # `scope["session"]` with the real, live ``Session`` and
996 # `setdefault` is a no-op — `_render_branded_error_page` renders
997 # against its own detached copy of that live session either way,
998 # so nothing here needs to know which case it is.
999 # `_render_branded_error_page` already catches every exception
1000 # itself and returns ``None`` on failure — `dict(scope)` and
1001 # `Request(...)` do no work that can raise — so there is nothing
1002 # left here to catch; rendering failures keep the plain-text
1003 # fallback via that ``None`` return. Unmatched and decorated
1004 # routes can skip the outer global rate-limit check, so rendering
1005 # here must not rely on an endpoint's rate-limit decorator having
1006 # run.
1007 req_scope = dict(scope)
1008 req_scope.setdefault("session", {})
1009 response = _render_branded_error_page(Request(req_scope), 413)
1010 if response is not None:
1011 # Response.__call__ never reads `receive` for a non-streaming
1012 # response (only sends http.response.start/body), so the real
1013 # `receive` is safe to pass through unused.
1014 await response(scope, receive, send)
1015 return
1017 body = b"Request too large"
1018 content_type = b"text/plain; charset=utf-8"
1019 await send(
1020 {
1021 "type": "http.response.start",
1022 "status": 413,
1023 "headers": [
1024 (b"content-type", content_type),
1025 (b"content-length", str(len(body)).encode()),
1026 ],
1027 }
1028 )
1029 await send({"type": "http.response.body", "body": body})
1031 async def __call__(self, scope, receive, send):
1032 if scope["type"] != "http":
1033 await self.app(scope, receive, send)
1034 return
1036 # Multipart ON one of the two real upload paths keeps the large
1037 # cap; everything else — JSON, a mislabelled multipart body, and
1038 # anything with a missing/other Content-Type, since a route can
1039 # call `request.json()` no matter what Content-Type was sent — is
1040 # additionally bounded by
1041 # the much smaller JSON cap. `min()` also means an explicit,
1042 # smaller `max_body_size` passed to the constructor (as tests
1043 # do) still wins, so this never widens an existing caller's cap.
1044 content_type = b""
1045 for name, value in scope.get("headers", []):
1046 if name == b"content-type":
1047 content_type = value
1048 break
1049 # Both conditions are required: a multipart *label* is not enough,
1050 # because it is client-supplied and costs nothing to forge (see
1051 # _MULTIPART_UPLOAD_PATHS). Only the real upload routes get the
1052 # upload-sized cap; a mislabelled body on any other path falls
1053 # through to the JSON cap below, exactly as an honestly-labelled
1054 # one does.
1055 is_multipart = content_type.lower().startswith(
1056 b"multipart/"
1057 ) and _is_multipart_upload_path(scope.get("path", ""))
1058 if is_multipart:
1059 effective_max = self.max_body_size
1060 else:
1061 # Routes that legitimately carry a large JSON body keep the
1062 # larger cap. Only notes qualifies: NOTE_CONTENT_MAX_BYTES is
1063 # 50 MB, and notes.py::_MAX_JSON_BODY_BYTES doubles that to
1064 # allow for JSON escaping. Every other `request.json()` call
1065 # site in the app carries a small query/config/message payload
1066 # — the largest realistic one is library_delete's bulk
1067 # `document_ids`, ~4 MB even at 100k UUIDs.
1068 json_cap = (
1069 self.max_large_json_body_size
1070 if scope.get("path", "").startswith(_LARGE_JSON_BODY_PREFIXES)
1071 else self.max_json_body_size
1072 )
1073 effective_max = min(self.max_body_size, json_cap)
1075 # Fast path: client declared an over-cap Content-Length.
1076 for name, value in scope.get("headers", []):
1077 if name == b"content-length":
1078 try:
1079 declared = int(value)
1080 except ValueError:
1081 break
1082 if declared > effective_max:
1083 await self._send_413(scope, receive, send)
1084 return
1085 break
1087 received = 0
1088 response_started = False
1090 async def recv_limited():
1091 nonlocal received
1092 message = await receive()
1093 if message["type"] == "http.request":
1094 received += len(message.get("body", b""))
1095 if received > effective_max:
1096 raise _RequestBodyTooLarge()
1097 return message
1099 async def send_tracking(message):
1100 nonlocal response_started
1101 if message["type"] == "http.response.start":
1102 response_started = True
1103 await send(message)
1105 try:
1106 await self.app(scope, recv_limited, send_tracking)
1107 except _RequestBodyTooLarge:
1108 if response_started:
1109 # Too late for a 413 — let the server tear the
1110 # connection down.
1111 raise
1112 from .routers.notes import _log_value_preview
1114 # (#6938: the path is client-chosen and this runs before
1115 # authentication, so it is bounded.)
1116 logger.warning(
1117 "Rejected over-limit request body on {} (> {} bytes)",
1118 _log_value_preview(scope.get("path", "")),
1119 effective_max,
1120 )
1121 await self._send_413(scope, receive, send)
1124class SecureCookieMiddleware:
1125 """Pure ASGI middleware to add Secure flag to cookies based on client IP.
1127 Ports Flask's SecureCookieMiddleware to ASGI.
1128 """
1130 def __init__(self, app, testing: bool = False):
1131 self.app = app
1132 self.testing = testing
1133 self._warned_insecure_public = False
1134 self._warned_untrusted_forwarded_proto = False
1135 self._warned_noncanonical_forwarded_proto = False
1137 def _maybe_warn_untrusted_forwarded_proto(
1138 self, scope, remote_addr: str
1139 ) -> None:
1140 """One-shot warning for the reverse-proxy misconfiguration that is
1141 otherwise completely silent.
1143 ``_maybe_warn_insecure_public`` below only fires for a NON-private
1144 client IP, so the most common broken upgrade never trips it: nginx (or
1145 caddy/traefik) terminating TLS on 127.0.0.1 with TRUST_PROXY_HEADERS
1146 unset. The peer address is loopback, so that warning takes the
1147 private-IP branch and says nothing, while uvicorn — not being told to
1148 honour forwarded headers — reports scheme "http". The session cookie
1149 then silently loses ``Secure`` and HSTS is silently withheld, on a
1150 deployment the operator believes is HTTPS.
1152 The tell is a request that carries ``X-Forwarded-Proto: https`` while
1153 the connection is not being treated as HTTPS. Cheap: this only runs on
1154 the not-https path (a correctly configured deployment never reaches
1155 it), and logs each distinct misconfiguration once. Until both warnings
1156 have been emitted, each request must be checked: a headerless health
1157 check or an untrusted direct client must not conceal a later proxy
1158 request.
1159 """
1160 if (
1161 self._warned_untrusted_forwarded_proto
1162 and self._warned_noncanonical_forwarded_proto
1163 ):
1164 return
1166 # Mirror uvicorn's ProxyHeadersMiddleware exactly (0.52.x): it reads
1167 # the LAST X-Forwarded-Proto header line, decodes it as latin-1,
1168 # strips it, and honours it only on an exact match against
1169 # {"http", "https", "ws", "wss"}. A value LDR would read as https
1170 # but uvicorn would not ("HTTPS", "https, https" from a proxy that
1171 # appends, "https,http") is a different misconfiguration from an
1172 # untrusted proxy and needs different advice -- in that case
1173 # uvicorn may well trust the proxy and have rewritten
1174 # scope["client"] from X-Forwarded-For already.
1175 proto_value = None
1176 for name, value in scope.get("headers", []):
1177 if name == b"x-forwarded-proto":
1178 proto_value = value
1179 if proto_value is None:
1180 return
1181 uvicorn_view = proto_value.decode("latin-1").strip()
1183 # No client address is ever interpolated into this advice. The
1184 # address in scope["client"] is not necessarily the proxy: with
1185 # TRUST_PROXY_HEADERS=true uvicorn may already have replaced it with
1186 # an X-Forwarded-For entry (under "*" the client-forgeable left-most
1187 # one, unvalidated -- "8.8.8.8/32" or free text), and even when it
1188 # is the raw socket peer, that peer may be an unlisted co-tenant
1189 # rather than the operator's proxy. Naming it as "the address to
1190 # allow-list" would let a client plant its own address into the
1191 # operator's forwarding trust.
1192 if uvicorn_view == "https":
1193 if self._warned_untrusted_forwarded_proto:
1194 return
1195 # uvicorn would have honoured this exact value had it trusted
1196 # the peer, so it did not: TRUST_PROXY_HEADERS is off, or the
1197 # peer is outside LDR_SERVER_FORWARDED_ALLOW_IPS. This header is
1198 # untrusted client input, and the advice here widens forwarding
1199 # trust, so give it only for a loopback/private peer: any remote
1200 # client could otherwise send `X-Forwarded-Proto: https` once
1201 # and plant a log line telling the operator to set
1202 # TRUST_PROXY_HEADERS=true -- advice that, followed on a host
1203 # NOT behind a proxy, would hand that same client control of the
1204 # scheme and therefore of the Secure-cookie and HSTS decisions.
1205 if not remote_addr or not _is_private_ip(remote_addr):
1206 return
1207 self._warned_untrusted_forwarded_proto = True
1208 logger.warning(
1209 "Received X-Forwarded-Proto: https but this connection is "
1210 "not being treated as HTTPS, so session cookies will NOT "
1211 "get the Secure flag and HSTS will NOT be sent. If this "
1212 "server sits behind a TLS-terminating reverse proxy, set "
1213 "TRUST_PROXY_HEADERS=true so forwarded headers are "
1214 "honoured. If it is already set, uvicorn does not trust the "
1215 "proxy that sent this request: add your reverse proxy's own "
1216 "address as LDR sees it (the address the proxy connects "
1217 "from, not a client address; with LDR in Docker behind a "
1218 "proxy on the host, the Docker network's gateway, not "
1219 "127.0.0.1) to LDR_SERVER_FORWARDED_ALLOW_IPS, which must "
1220 "list every proxy hop in front of LDR."
1221 )
1222 elif uvicorn_view.split(",")[0].strip().lower() == "https":
1223 if self._warned_noncanonical_forwarded_proto:
1224 return
1225 # uvicorn ignores this value whether or not it trusts the peer,
1226 # so neither TRUST_PROXY_HEADERS nor the allow-list is the fix,
1227 # and the advice (reconfigure the proxy) widens no trust -- it
1228 # is safe to give for any peer, including one uvicorn has
1229 # already rewritten to a public client address.
1230 self._warned_noncanonical_forwarded_proto = True
1231 logger.warning(
1232 "Received an X-Forwarded-Proto header claiming https in a "
1233 "form uvicorn does not accept, so this connection is not "
1234 "being treated as HTTPS: session cookies will NOT get the "
1235 "Secure flag and HSTS will NOT be sent. uvicorn honours "
1236 "only a single exact lower-case value (http, https, ws or "
1237 "wss) in the last X-Forwarded-Proto header, so 'HTTPS' or "
1238 "a comma-separated list such as 'https, https' is "
1239 "ignored. If a TLS-terminating reverse proxy sets this "
1240 "header, configure it to overwrite X-Forwarded-Proto with "
1241 "exactly 'https' rather than appending to it (nginx: "
1242 "proxy_set_header X-Forwarded-Proto $scheme;)."
1243 )
1245 def _maybe_warn_insecure_public(self, remote_addr: str) -> None:
1246 """One-shot warning when serving HTTP to a non-private client IP —
1247 a likely missing HTTPS proxy. Mirrors main's operator signal."""
1248 if self._warned_insecure_public:
1249 return
1250 if remote_addr and not _is_private_ip(remote_addr):
1251 self._warned_insecure_public = True
1252 from ..security.log_sanitizer import redact_and_bound_for_log
1254 logger.warning(
1255 "Serving HTTP to a public client ({}). Session/CSRF cookies "
1256 "are NOT marked Secure (marking them Secure over HTTP would "
1257 "make the browser drop them). Put LDR behind HTTPS, and if "
1258 "using a TLS proxy set TRUST_PROXY_HEADERS=true.",
1259 # remote_addr parsed as an IP (non-IPs count as private),
1260 # but ipaddress accepts an IPv6 zone id of arbitrary text
1261 # ("2001:db8::1%any words"), and with TRUST_PROXY_HEADERS
1262 # uvicorn may have copied it from X-Forwarded-For. Log the
1263 # address alone.
1264 redact_and_bound_for_log(remote_addr.partition("%")[0], 64),
1265 )
1267 async def __call__(self, scope, receive, send):
1268 if scope["type"] != "http":
1269 await self.app(scope, receive, send)
1270 return
1272 # Add the Secure flag ONLY when the connection is actually HTTPS.
1273 # Matches Flask's SecureCookieMiddleware (#3849): setting Secure on a
1274 # cookie served over plain HTTP makes the browser DROP the cookie
1275 # (RFC6265bis), so a deployment reachable over HTTP from a public IP
1276 # would hit an unbreakable login loop. A previous version here added
1277 # Secure for "public IP over HTTP" too, reintroducing exactly that
1278 # bug. For a deployment behind a TLS-terminating proxy, honor
1279 # X-Forwarded-Proto by starting uvicorn with TRUST_PROXY_HEADERS=true
1280 # so scope["scheme"] reflects https (see web/app.py). We still warn
1281 # once when serving HTTP to a non-private client so a missing HTTPS
1282 # proxy is visible.
1283 client = scope.get("client")
1284 remote_addr = client[0] if client else ""
1285 is_https = scope.get("scheme") == "https"
1286 if not is_https and not self.testing:
1287 self._maybe_warn_insecure_public(remote_addr)
1288 self._maybe_warn_untrusted_forwarded_proto(scope, remote_addr)
1289 should_add_secure = not self.testing and is_https
1291 if not should_add_secure:
1292 await self.app(scope, receive, send)
1293 return
1295 async def send_wrapper(message):
1296 if message["type"] == "http.response.start":
1297 headers = []
1298 for k, v in message.get("headers", []):
1299 if k.lower() == b"set-cookie":
1300 v_str = v.decode("latin-1")
1301 if "; Secure" not in v_str and "; secure" not in v_str:
1302 v_str += "; Secure"
1303 v = v_str.encode("latin-1")
1304 headers.append((k, v))
1305 message = {**message, "headers": headers}
1306 await send(message)
1308 await self.app(scope, receive, send_wrapper)
1311class RememberMeMiddleware:
1312 """Strip session-cookie Max-Age/Expires unless `_remember_me` is True.
1314 Starlette's SessionMiddleware accepts a single `max_age` applied to
1315 every session cookie. Without this middleware, every login gets a
1316 30-day persistent cookie regardless of whether the user ticked
1317 "Remember me" — which is the usual UX complaint, and would be a
1318 privacy regression vs. the Flask app that honored the flag (Flask
1319 defaulted `session.permanent = False`, i.e. a browser-session
1320 cookie, and only made it persistent when "remember me" was
1321 checked).
1323 The login handler stores `_remember_me` inside the session dict.
1324 On every response we inspect it and, unless it is explicitly
1325 `True`, strip `Max-Age` and `Expires` attributes from the outbound
1326 `session=...` Set-Cookie so the browser treats it as a session
1327 cookie (discarded on browser close). Anonymous visitors (no login
1328 yet) and requests right after logout also read back as "not True"
1329 here — `_remember_me` is either absent (never logged in) or wiped
1330 by `request.session.clear()` at logout (routers/auth.py), so it
1331 reads back as `None`, not `False`. Both must get the same
1332 non-persistent cookie as an explicit `_remember_me=False` login;
1333 checking `remember_me is False` instead of `is not True` would miss
1334 both cases and hand every anonymous visitor and every just-logged-out
1335 user a persistent 30-day cookie.
1337 This ONLY relaxes the client-side (browser) cap. The itsdangerous
1338 signature backing the cookie is still valid for the full
1339 `security.session_remember_me_days` window (SessionMiddleware has a
1340 single `max_age` shared by every session — see its construction
1341 below) regardless of this flag, so a raw non-remember-me cookie value
1342 copied out of the browser (XSS exfiltration, proxy log, devtools)
1343 would replay successfully for up to 30 days even though the browser
1344 itself was told to discard it on close. `_enforce_session_expiry` /
1345 `_stamp_session_expiry` (used by `DatabaseMiddleware` below) close
1346 that gap by enforcing the shorter `security.session_timeout_hours`
1347 window server-side, independent of the browser-facing cookie
1348 attributes this middleware controls.
1350 EXCEPTION — the deletion cookie must survive untouched. When a session
1351 is cleared server-side (idle-timeout revocation in
1352 `_enforce_session_revocation`, or logout), Starlette's SessionMiddleware
1353 does not write a normal data cookie: it writes the literal
1354 `session=null; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT; ...`
1355 (see `starlette.middleware.sessions.SessionMiddleware.send_wrapper`,
1356 the `session.modified and not initial_session_was_empty` branch) — an
1357 explicit "delete this cookie now" instruction, not persistent session
1358 data. Stripping `Expires`/`Max-Age` from THAT header, the same as any
1359 other non-remember-me cookie, downgrades it to `session=null; path=/;
1360 httponly; samesite=lax`: no expiry in the past, so the browser does
1361 not delete the cookie at all — it just keeps sending the literal string
1362 "null" back as the session value for the rest of the browser session.
1363 A revoked/logged-out session would then never actually leave the
1364 client. `session=null;` is the exact, fixed marker Starlette emits for
1365 a cleared session (never a signed payload), so matching it precisely
1366 identifies the deletion cookie without needing to duplicate
1367 SessionMiddleware's own emptiness bookkeeping.
1368 """
1370 # Name of the cookie Starlette's SessionMiddleware sets.
1371 _SESSION_COOKIE_NAME = b"session"
1373 # The literal value Starlette's SessionMiddleware writes when a session
1374 # is cleared (see the class docstring's EXCEPTION section). Distinct by
1375 # construction from every real session cookie, which carries a base64
1376 # itsdangerous-signed payload, never the bare string "null".
1377 _DELETED_COOKIE_PREFIX = _SESSION_COOKIE_NAME + b"=null;"
1379 def __init__(self, app) -> None:
1380 self.app = app
1382 async def __call__(self, scope, receive, send):
1383 if scope["type"] != "http":
1384 await self.app(scope, receive, send)
1385 return
1387 async def send_wrapper(message):
1388 if message["type"] != "http.response.start":
1389 await send(message)
1390 return
1392 # Resolve remember_me preference from the session. Treat
1393 # anything other than an explicit True (missing/None for
1394 # anonymous or just-logged-out sessions, or False for an
1395 # explicit non-remember-me login) as "strip the persistent
1396 # cookie attributes" — only an explicit opt-in keeps them.
1397 session = scope.get("session", {}) or {}
1398 remember_me = session.get("_remember_me")
1399 if remember_me is not True:
1400 rewritten = []
1401 for k, v in message.get("headers", []):
1402 if (
1403 k.lower() == b"set-cookie"
1404 and v.lower().startswith(
1405 self._SESSION_COOKIE_NAME + b"="
1406 )
1407 and not v.lower().startswith(
1408 self._DELETED_COOKIE_PREFIX
1409 )
1410 ):
1411 v_str = v.decode("latin-1")
1412 # Drop Max-Age=... and Expires=... attributes.
1413 parts = [
1414 part
1415 for part in v_str.split(";")
1416 if not part.strip()
1417 .lower()
1418 .startswith(("max-age=", "expires="))
1419 ]
1420 v = ";".join(parts).encode("latin-1")
1421 rewritten.append((k, v))
1422 message = {**message, "headers": rewritten}
1423 await send(message)
1425 await self.app(scope, receive, send_wrapper)
1428# ---------------------------------------------------------------------------
1429# Exception handlers
1430# ---------------------------------------------------------------------------
1433def _is_api_request(request: Request) -> bool:
1434 """Check if this is an API/JSON request (vs browser HTML request)."""
1435 path = request.url.path
1436 # Match paths containing /api/ segment or ending with /api
1437 if "/api/" in path or path.endswith("/api"):
1438 return True
1439 accept = request.headers.get("accept", "")
1440 if "application/json" in accept and "text/html" not in accept:
1441 return True
1442 return False
1445def _register_exception_handlers(app: FastAPI) -> None:
1446 """Register custom exception handlers (ports Flask's error handlers)."""
1448 from fastapi.exceptions import HTTPException, RequestValidationError
1450 from .exceptions import WebAPIException
1452 # Pydantic's ``msg`` can contain submitted values when a validator raises
1453 # ValueError or PydanticCustomError. Build the browser summary and the
1454 # structured detail from fixed messages keyed by error type instead.
1455 safe_validation_messages = {
1456 "missing": "Field required",
1457 "extra_forbidden": "Unexpected field",
1458 "json_invalid": "Invalid JSON",
1459 "int_parsing": "Enter a valid integer",
1460 "int_type": "Enter a valid integer",
1461 "float_parsing": "Enter a valid number",
1462 "float_type": "Enter a valid number",
1463 "bool_parsing": "Enter true or false",
1464 "bool_type": "Enter true or false",
1465 "string_type": "Enter text",
1466 "string_too_short": "Text is too short",
1467 "string_too_long": "Text is too long",
1468 "greater_than": "Value is too small",
1469 "greater_than_equal": "Value is too small",
1470 "less_than": "Value is too large",
1471 "less_than_equal": "Value is too large",
1472 "literal_error": "Choose an allowed value",
1473 "list_type": "Enter a list",
1474 "dict_type": "Enter an object",
1475 }
1477 @app.exception_handler(RequestValidationError)
1478 async def handle_request_validation_error(
1479 request: Request, exc: RequestValidationError
1480 ):
1481 # Keep ``loc``/``type`` for API clients, replace every ``msg`` with
1482 # fixed text, drop ``input``/``ctx``/``url``, and add the ``error``
1483 # string consumed by the browser. Validator messages and submitted
1484 # values are never forwarded. ``loc`` is copied as-is, so it can still
1485 # echo submitted dict keys (no current route has a dict-typed or
1486 # extra=forbid body); for json_invalid it ends in a parser offset.
1487 errors = [
1488 {
1489 "loc": error["loc"],
1490 "msg": safe_validation_messages.get(
1491 error["type"], "Invalid value"
1492 ),
1493 "type": error["type"],
1494 }
1495 for error in exc.errors()
1496 ]
1497 messages = []
1498 for error in errors[:3]:
1499 location = error["loc"]
1500 if location and location[0] in ( 1500 ↛ 1508line 1500 didn't jump to line 1508 because the condition on line 1500 was always true
1501 "body",
1502 "query",
1503 "path",
1504 "header",
1505 "cookie",
1506 ):
1507 location = location[1:]
1508 field = ".".join(str(part) for part in location)
1509 if error["type"] == "json_invalid" and location:
1510 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: JSON error position is a response label and never reaches SQL
1511 field = f"position {location[-1]}"
1512 messages.append(
1513 f"{field}: {error['msg']}" if field else error["msg"]
1514 )
1515 message = "Invalid request"
1516 if messages: 1516 ↛ 1518line 1516 didn't jump to line 1518 because the condition on line 1516 was always true
1517 message += ": " + "; ".join(messages)
1518 if len(errors) > 3: 1518 ↛ 1519line 1518 didn't jump to line 1519 because the condition on line 1518 was never true
1519 message += f"; and {len(errors) - 3} more validation errors"
1520 return JSONResponse(
1521 {"error": message, "detail": errors}, status_code=422
1522 )
1524 @app.exception_handler(HTTPException)
1525 async def handle_http_exception(request: Request, exc: HTTPException):
1526 """Redirect 401s for browser requests to login page (Flask parity)."""
1527 if exc.status_code == 401 and not _is_api_request(request):
1528 login_url = "/auth/login"
1529 # Path AND query. main used Werkzeug's `request.url` (the full
1530 # URL) as `next`, so a deep link survived the login bounce; the
1531 # port truncated it to the path, silently dropping the query.
1532 # That is not cosmetic: a signed-out user following a link such
1533 # as /settings/?tab=llm lost everything after the `?`. A chat
1534 # page's `q` survives this bounce too, but not the login: the
1535 # login page drops it from `next` (routers/auth.py:
1536 # _drop_chat_query), because after login the browser follows
1537 # our own redirect and the chat page cannot tell that another
1538 # site supplied the text. (The chat route also drops `q` before
1539 # this bounce when Sec-Fetch-Site marks the link as coming from
1540 # another origin.) The consumer side already
1541 # handles this -- URLValidator.get_safe_redirect_path()
1542 # explicitly re-appends query and fragment, and is byte-identical
1543 # to main. quote(safe="/") below encodes the `?` and `&` so the
1544 # embedded value stays a single well-formed query parameter.
1545 next_url = str(request.url.path)
1546 if request.url.query:
1547 next_url += "?" + request.url.query
1548 if next_url and next_url != "/":
1549 # URL-encode the path before embedding as a query value.
1550 # Without quote(), characters like `?` or `&` in the path
1551 # break the resulting URL; downstream code that treats
1552 # ?next= as untrusted input must still validate it's a
1553 # local path, but at minimum the encoding here keeps the
1554 # redirect target well-formed.
1555 from urllib.parse import quote
1557 login_url += f"?next={quote(next_url, safe='/')}"
1558 return RedirectResponse(
1559 url=login_url,
1560 status_code=302,
1561 headers=exc.headers,
1562 )
1563 path = request.url.path
1564 if path == "/api/v1" or path.startswith("/api/v1/"):
1565 # /api/v1 is the documented programmatic API for external clients,
1566 # and main returned {"error": ...} for its auth failures
1567 # (web/api.py's `jsonify({"error": "Authentication required"})` /
1568 # `{"error": "API access is disabled"}`). Routing those through
1569 # HTTPException here would silently change the envelope to
1570 # {"detail": ...} for every existing script.
1571 #
1572 # Both keys are emitted rather than swapping: the frontend's
1573 # api.js reads `detail`, and tests/web/test_exception_handler_
1574 # contract.py deliberately pins the {"detail": ...} shape for the
1575 # rest of the app. This restores main's contract for the external
1576 # API without disturbing either.
1577 return JSONResponse(
1578 {"error": exc.detail, "detail": exc.detail},
1579 status_code=exc.status_code,
1580 headers=exc.headers,
1581 )
1582 return JSONResponse(
1583 {"detail": exc.detail},
1584 status_code=exc.status_code,
1585 headers=exc.headers,
1586 )
1588 @app.exception_handler(404)
1589 async def not_found(request: Request, exc):
1590 # Browser navigations get HTML; only API callers get JSON. Without
1591 # the branch, a user who mistypes a URL is shown a raw
1592 # ``{"error": "Not found"}`` body in the browser's JSON viewer.
1593 # Flask branched the same way — the retired app_factory.py's
1594 # ``@app.errorhandler(404)`` returned ``make_response("Not found",
1595 # 404)``, which Flask serves as text/html — and the 401 handler
1596 # directly above already makes this distinction. This one did not.
1597 if _is_api_request(request):
1598 return JSONResponse({"error": "Not found"}, status_code=404)
1599 # Branded page (PR #5424), falling back to the old fixed plain-text
1600 # body if rendering fails for any reason — see
1601 # _render_branded_error_page's docstring.
1602 page = _render_branded_error_page(request, 404)
1603 if page is not None:
1604 return page
1605 return HTMLResponse("Not found", status_code=404)
1607 # Catch-all so unhandled exceptions get logged with traceback rather than
1608 # silently disappearing into Starlette's default 500 path. (The Exception
1609 # handler takes precedence over status-code 500 via MRO, so a separate
1610 # 500 handler would be unreachable.)
1611 @app.exception_handler(Exception)
1612 async def unhandled_exception(request: Request, exc):
1613 from fastapi import HTTPException
1615 # Let HTTPException pass through to FastAPI's default handler.
1616 if isinstance(exc, HTTPException): 1616 ↛ 1617line 1616 didn't jump to line 1617 because the condition on line 1616 was never true
1617 raise exc from None
1618 from .routers.notes import _log_value_preview
1620 # (#6938: the path is client-chosen, so it is bounded.)
1621 logger.opt(exception=exc).error(
1622 "Unhandled exception: {} {}",
1623 request.method,
1624 _log_value_preview(request.url.path),
1625 )
1626 # Stamp the security headers here rather than relying on
1627 # SecurityHeadersMiddleware. Registering a handler for the bare
1628 # ``Exception`` class makes Starlette wire it into
1629 # ``ServerErrorMiddleware``, which is installed OUTSIDE every
1630 # ``add_middleware`` layer and writes to the raw ASGI ``send`` —
1631 # so this response never passes through that middleware and would
1632 # otherwise ship with no CSP, no nosniff, no frame-options — and,
1633 # same reasoning, no Cache-Control/Pragma/Expires either (main's
1634 # Flask ``after_request`` covered every response including
1635 # unhandled 500s; this reuses SecurityHeadersMiddleware.
1636 # cache_headers() so the two can't drift apart).
1637 # Preserve the fixed JSON 500 response for both browser and API
1638 # callers. Branding 404/413 does not change this error envelope.
1639 return JSONResponse(
1640 {"error": "Server error"},
1641 status_code=500,
1642 headers={
1643 name.decode(): value.decode()
1644 for name, value in SecurityHeadersMiddleware.response_headers(
1645 path=request.url.path,
1646 scheme=request.url.scheme,
1647 status=500,
1648 )
1649 },
1650 )
1652 # Flask parity: request.get_json() returned 400 on malformed JSON and on
1653 # request bodies that could not be decoded as UTF-8. Without these two
1654 # registrations a bare `await request.json()` surfaced either exception
1655 # as a 500 via the catch-all above.
1656 import json
1658 @app.exception_handler(UnicodeDecodeError)
1659 @app.exception_handler(json.JSONDecodeError)
1660 async def handle_json_decode_error(request: Request, exc):
1661 # Log it: this handler also catches a JSONDecodeError raised deeper
1662 # in a handler (e.g. parsing a downstream response), which is a
1663 # server-side fault wearing a 400. Without this line those would
1664 # vanish silently — the catch-all above logs everything it takes.
1665 #
1666 # Bounded fields only, never `exc` itself and never `exc.doc`.
1667 # JSONDecodeError carries the ENTIRE offending document on `.doc`,
1668 # so interpolating the exception is one __str__ change away from
1669 # putting a request body — or a downstream provider's response — in
1670 # the log. `.msg` is the parser's own text ("Expecting value"),
1671 # `.lineno`/`.colno` are ints. This also matches how every sibling
1672 # handler here logs (error_code / status_code / reason), and closes
1673 # a real blind spot in check-sensitive-logging, which tracks
1674 # exception variables via `except ... as e` and therefore cannot
1675 # see one arriving as an exception-handler parameter.
1676 from .routers.notes import _log_value_preview
1678 # (#6938: the path is client-chosen, so it is bounded.)
1679 logger.warning(
1680 "JSON decode error handling {} {}: {} (line {} column {})",
1681 request.method,
1682 _log_value_preview(request.url.path),
1683 getattr(exc, "msg", "invalid JSON"),
1684 getattr(exc, "lineno", "?"),
1685 getattr(exc, "colno", "?"),
1686 )
1687 return JSONResponse({"error": "Invalid JSON body"}, status_code=400)
1689 # A client that hangs up before its request body is complete surfaces
1690 # from ``await request.json()`` (and ``body()``/``stream()``/``form()``)
1691 # as ``ClientDisconnect``. ``read_json_dict`` lets it through on purpose
1692 # (its docstring has the reasoning), so without a handler here the
1693 # catch-all above reports it as a backend outage: an error-level log
1694 # with a traceback and a 500, for something that is the client's doing.
1695 #
1696 # Info level, no traceback, and a 4xx. 499 is nginx's "Client Closed
1697 # Request", the de-facto status for exactly this: unlike a 5xx it is not
1698 # counted as a server fault, and unlike 400 it cannot be mistaken for a
1699 # malformed body. The client is gone, so uvicorn drops whatever is sent
1700 # after the disconnect (and writes no access-log line for it); the
1701 # status is only ever seen in-process.
1702 #
1703 # Registered for ``ClientDisconnect`` alone. The over-cap exception
1704 # ``BodySizeLimitMiddleware`` raises is a different class and still
1705 # reaches that middleware, which logs the rejection and answers 413.
1706 @app.exception_handler(ClientDisconnect)
1707 async def handle_client_disconnect(request: Request, exc):
1708 from .routers.notes import _log_value_preview
1710 # Bounded fields only, as the decode-error handler above does. The
1711 # path is client-chosen (#6938), so it is bounded too.
1712 logger.info(
1713 "Client disconnected before the request body was complete: {} {}",
1714 request.method,
1715 _log_value_preview(request.url.path),
1716 )
1717 return JSONResponse({"error": "Client disconnected"}, status_code=499)
1719 @app.exception_handler(WebAPIException)
1720 async def handle_web_api_exception(request: Request, exc: WebAPIException):
1721 logger.error(
1722 "Web API error: {} (status {})", exc.error_code, exc.status_code
1723 )
1724 return JSONResponse(exc.to_dict(), status_code=exc.status_code)
1726 # A rekey operational fault that a route leaves unhandled: the user's
1727 # database could not be checked or opened because the rekey sidecar or
1728 # an epoch file could not be read (an fd-exhaustion spike, EACCES, EIO),
1729 # or its durable rekey state needs recovery first. get_user_db_session
1730 # raises it as DatabaseSessionUnavailableError; a db_manager call made
1731 # directly raises the RekeyOperationalError itself, and so does a
1732 # settings read through get_settings_manager without a session (it
1733 # reaches db_manager.get_session). Neither error is a verdict on the
1734 # session, so both get require_auth's retryable 503
1735 # (same body and Retry-After, through the HTTPException handler above)
1736 # instead of the catch-all's 500, and the session is left alone. Logged
1737 # by type name only: the frames the fault was raised through hold the
1738 # plaintext password, so no traceback is rendered.
1739 from ..database.rekey_lock import RekeyOperationalError
1740 from ..database.session_context import DatabaseSessionUnavailableError
1742 @app.exception_handler(DatabaseSessionUnavailableError)
1743 @app.exception_handler(RekeyOperationalError)
1744 async def handle_database_unavailable(request: Request, exc: Exception):
1745 from .dependencies.auth import database_unavailable_exception
1746 from .routers.notes import _log_value_preview
1748 # (#6938: the path is client-chosen, so it is bounded.)
1749 logger.warning(
1750 "Database unavailable while handling {} {} ({}); answering 503",
1751 request.method,
1752 _log_value_preview(request.url.path),
1753 exc.reason
1754 if isinstance(exc, DatabaseSessionUnavailableError)
1755 else type(exc).__name__,
1756 )
1757 return await handle_http_exception(
1758 request, database_unavailable_exception()
1759 )
1761 try:
1762 from ..news.exceptions import NewsAPIException
1764 @app.exception_handler(NewsAPIException)
1765 async def handle_news_api_exception(
1766 request: Request, exc: NewsAPIException
1767 ):
1768 logger.error(
1769 "News API error: {} (status {})",
1770 exc.error_code,
1771 exc.status_code,
1772 )
1773 return JSONResponse(exc.to_dict(), status_code=exc.status_code)
1774 except ImportError:
1775 pass
1777 # PolicyDeniedError can escape any synchronous request-path PEP
1778 # (LLM, embeddings, URL fetch) that wasn't caught at the call site.
1779 # Without this handler it surfaces as a 500 — turn it into a clean
1780 # 400 with the decision reason so the user (and the operator
1781 # audit log) sees a policy-shaped error instead of a stack trace.
1782 try:
1783 from ..security.egress.policy import PolicyDeniedError
1785 @app.exception_handler(PolicyDeniedError)
1786 async def handle_policy_denied(
1787 request: Request, exc: PolicyDeniedError
1788 ):
1789 reason = getattr(getattr(exc, "decision", None), "reason", "denied")
1790 target = getattr(exc, "target", "")
1791 logger.bind(policy_audit=True).warning(
1792 "PolicyDeniedError surfaced at FastAPI exception handler",
1793 reason=reason,
1794 target=target,
1795 )
1796 return JSONResponse(
1797 {
1798 "status": "error",
1799 "message": (
1800 f"Egress policy refused this request: {reason}"
1801 ),
1802 },
1803 status_code=400,
1804 )
1805 except ImportError:
1806 # egress_policy is in-tree; skipping the handler is only useful
1807 # for partial test builds without the security package.
1808 pass
1810 # Strict settings reads raise SettingsValueError for an invalid
1811 # configured value. One that reaches the app gets its diagnostic, which
1812 # names the key, its source and a fixed reason but never the value, so
1813 # the operator can find the setting. Without this handler the catch-all
1814 # above would answer with a generic "Server error".
1815 from ..settings.exceptions import SettingsValueError
1817 @app.exception_handler(SettingsValueError)
1818 async def handle_settings_value_error(
1819 request: Request, exc: SettingsValueError
1820 ):
1821 from .routers.notes import _log_value_preview
1823 diagnostic = exc.diagnostic
1824 logger.error(
1825 "Invalid setting on {} {}: {}",
1826 request.method,
1827 _log_value_preview(request.url.path),
1828 diagnostic,
1829 )
1830 if _is_api_request(request):
1831 return JSONResponse({"error": diagnostic}, status_code=500)
1832 page = _render_branded_error_page(
1833 request,
1834 500,
1835 title="Invalid setting",
1836 message=(
1837 f"{diagnostic}. Correct the setting in the settings page "
1838 "or the deployment environment, or remove it, then reload."
1839 ),
1840 )
1841 if page is not None:
1842 return page
1843 return PlainTextResponse(diagnostic, status_code=500)
1846# ---------------------------------------------------------------------------
1847# Database middleware (before each request)
1848# ---------------------------------------------------------------------------
1850# Regression #1 fix — see _enforce_session_expiry's docstring for the full
1851# mechanism. Read once at import time, matching how SessionMiddleware's own
1852# `max_age` below is computed: an operator change to
1853# security.session_timeout_hours takes effect on the next restart, not live.
1854_NON_REMEMBER_ME_SESSION_SECONDS = (
1855 get_security_default("security.session_timeout_hours", 2) * 3600
1856)
1858_SESSION_EXPIRES_AT_KEY = "_session_expires_at"
1861def _now_ts() -> float:
1862 """Wall-clock now, as epoch seconds.
1864 A thin wrapper around ``time.time()`` purely so tests can monkeypatch
1865 session-expiry time (``local_deep_research.web.fastapi_app._now_ts``)
1866 without touching the process-global stdlib clock.
1867 """
1868 return time.time()
1871def _enforce_session_expiry(session) -> None:
1872 """Clear `session` in place if its server-side deadline has passed.
1874 Regression #1: Starlette's ``SessionMiddleware`` verifies the
1875 itsdangerous signature with a single ``max_age`` shared by every
1876 session (see its construction below) — the signer has no notion of
1877 "this particular session should expire sooner." That means a
1878 non-remember-me login's raw cookie value stays independently
1879 replayable for the FULL ``security.session_remember_me_days`` window
1880 (30 days) purely because that's what SessionMiddleware was built
1881 with, even though the browser is told (by ``RememberMeMiddleware``)
1882 to discard the cookie on close. The intended lifetime for a
1883 non-remember-me session is the much shorter
1884 ``security.session_timeout_hours`` (default 2h).
1886 Mechanism chosen: stamp an expiry deadline into the session payload
1887 itself (`_stamp_session_expiry` below) and enforce it here, in
1888 application code, independent of itsdangerous's own signature check.
1890 Alternatives considered and rejected:
1891 * Two ``SessionMiddleware`` instances (short/long ``max_age``),
1892 dispatched by ``_remember_me`` — the dispatch decision itself
1893 requires reading the session, i.e. doing the same "peek before you
1894 verify" work this function already does, for substantially more
1895 code (two secrets or cookie names to keep in sync, or a
1896 request-splitting wrapper ASGI app in front of both).
1897 * A custom itsdangerous signer with a variable ``max_age`` — ``max_age``
1898 is a verification-time argument to ``signer.unsign()``, not part of
1899 the signed payload, so "vary the signer" still needs some per-session
1900 signal to pick which ``max_age`` to verify with BEFORE verification
1901 succeeds. That signal is exactly the deadline stamped here — no
1902 simpler, and itsdangerous's own timestamp isn't readable without a
1903 successful unsign first (chicken-and-egg).
1904 * Enforce the same shorter deadline on remember-me sessions too
1905 (uniform handling, less code) — rejected: their itsdangerous-level
1906 cap already matches ``security.session_remember_me_days`` exactly,
1907 so adding a second, redundant deadline only risks the two drifting
1908 apart if an operator changes one setting and not the other.
1910 Called from ``DatabaseMiddleware.__call__`` after CSRFMiddleware and
1911 before Starlette's router. CSRF can still inspect a signed but expired
1912 session, but no route handler (including ``require_auth`` / ``/auth/check``)
1913 can act on one whose intended lifetime has passed. Clearing the dict
1914 here makes an expired session indistinguishable from "never logged in"
1915 to downstream handlers without touching those call sites.
1917 Existing sessions: does NOT force-log-out non-remember-me sessions
1918 that predate this fix. Their cookie has no stamped deadline yet
1919 (`_session_expires_at` absent is treated as "not yet due", not
1920 "expired"), so this is a no-op for them on their first request after
1921 deploy — `_stamp_session_expiry` gives them a fresh deadline on that
1922 same response instead. The one residual gap this leaves: a
1923 non-remember-me cookie captured (XSS/log/devtools) in the narrow
1924 window between deploy and that session's next request keeps its old,
1925 unbounded-until-30-days replay property for one more grant of
1926 ``security.session_timeout_hours``. A hard cutover (immediate mass
1927 logout of every non-remember-me session) would close that but was
1928 rejected per this task's explicit instruction not to log everyone out
1929 unnecessarily.
1930 """
1931 if session.get("_remember_me") is not False:
1932 return
1933 expires_at = session.get(_SESSION_EXPIRES_AT_KEY)
1934 if expires_at is not None and _now_ts() >= expires_at:
1935 session.clear()
1938def _stamp_session_expiry(session) -> None:
1939 """(Re)stamp the non-remember-me deadline `_enforce_session_expiry`
1940 checks, sliding it forward on every response for an authenticated,
1941 non-remember-me session. No-op for anonymous or remember-me sessions.
1943 Runs from `DatabaseMiddleware`'s response-phase `send_wrapper` — i.e.
1944 after any route handler (including the login handler itself) has
1945 already run — so the very first Set-Cookie a non-remember-me login
1946 produces already carries a deadline, not just its second response
1947 onward. Sliding (refreshed on every authenticated response, like
1948 ``session_manager.SessionManager``'s ``last_access``-based timeout)
1949 rather than a fixed deadline from login time, matching
1950 ``security.session_timeout_hours``'s documented meaning: "Session
1951 timeout ... for sessions without 'Remember Me' checked" — an idle
1952 timeout, not a hard cap on total session length.
1953 """
1954 if session.get("username") and session.get("_remember_me") is False:
1955 session[_SESSION_EXPIRES_AT_KEY] = (
1956 _now_ts() + _NON_REMEMBER_ME_SESSION_SECONDS
1957 )
1960def _enforce_session_revocation(session) -> None:
1961 """Clear `session` in place if its server-side record is gone.
1963 `require_auth` rejects a cookie whose `session_id` no longer resolves
1964 to the claimed username, which is what makes logout actually revoke a
1965 session rather than merely ask the browser to forget it. When this was
1966 added, two GET routes bypassed `require_auth` — `/` and `/auth/check` —
1967 because they must render/answer for anonymous visitors too, and both
1968 read `request.session["username"]` directly. A sweep of all 110 GET
1969 routes found exactly those two granting a revoked session strictly
1970 more than an anonymous one: after logout, a replayed cookie still
1971 rendered the authenticated index and still got `authenticated: true`,
1972 for the full remaining itsdangerous window. Both now call
1973 `require_auth` themselves (catching its 401), which also applies the
1974 per-session credential check this function does not perform.
1976 Enforcing revocation here as well means the property holds for every
1977 route by construction — including any future handler that reads the
1978 session without going through `require_auth`, which is precisely how
1979 these two got missed. Same placement rationale
1980 as `_enforce_session_expiry`: clearing the dict makes a revoked
1981 session indistinguishable from "never logged in" to everything
1982 downstream, so no call site needs to know about revocation.
1984 Anonymous sessions return immediately, so the cost on the hot path is
1985 one dict lookup; for authenticated ones it is `validate_session`'s
1986 in-memory dict read under a lock. That call also refreshes
1987 `last_access`, which is why this runs unconditionally rather than
1988 behind `_skip_prefixes` — same reasoning as the expiry check above.
1990 Note `session_manager.sessions` is in-memory, so a server restart
1991 invalidates every session. That is pre-existing `require_auth`
1992 behaviour, not something this widens: it already applied to every
1993 authenticated route. The only change is that `/` and `/auth/check`
1994 now agree with the rest of the app about who is logged in.
1995 """
1996 username = session.get("username")
1997 if not username:
1998 return
2000 from .auth.session_manager import session_manager
2002 session_id = session.get("session_id")
2003 if not session_id or session_manager.validate_session(session_id) != (
2004 username
2005 ):
2006 session.clear()
2009class DatabaseMiddleware:
2010 """Pure ASGI middleware to ensure user database is open for authenticated requests.
2012 Runs INSIDE SessionMiddleware so request.session is available. Also
2013 enforces/refreshes the non-remember-me session deadline (regression
2014 #1 — see `_enforce_session_expiry` / `_stamp_session_expiry` above)
2015 since this is the innermost app-level middleware: the last point to
2016 act on the session before any route handler runs. CSRFMiddleware runs
2017 outside it and reads the session first.
2018 """
2020 _skip_prefixes = (
2021 "/static/",
2022 "/favicon.ico",
2023 "/api/v1/health",
2024 "/auth/login",
2025 "/auth/register",
2026 "/auth/csrf-token",
2027 "/ws/",
2028 )
2030 def __init__(self, app):
2031 self.app = app
2033 async def __call__(self, scope, receive, send):
2034 if scope["type"] != "http":
2035 await self.app(scope, receive, send)
2036 return
2038 # One scope per request around everything below: the reopen of the
2039 # user's database, require_auth's probes and the route's own
2040 # database opens share a give-up flag for their waits on the user's
2041 # rekey sidecar (a wait that gives up sets it, and each later claim
2042 # then tries once without waiting, until one of them gets the
2043 # sidecar again; it is not a running total of the time waited), and
2044 # require_auth sees whether the reopen met a rekey fault (a 503
2045 # that keeps the session, not a 401). See request_database_scope.
2046 from .dependencies.auth import request_database_scope
2048 with request_database_scope():
2049 await self._call_http(scope, receive, send)
2051 async def _call_http(self, scope, receive, send):
2052 path = scope.get("path", "")
2053 method = scope.get("method", "GET")
2055 from ..utilities.request_context import (
2056 reset_request_user,
2057 set_request_user,
2058 )
2060 # Regression #1: enforce the shorter non-remember-me deadline
2061 # BEFORE the route handler can act on a session whose itsdangerous
2062 # signature is still valid but whose intended lifetime has passed.
2063 # CSRFMiddleware already read the session before this point. Runs
2064 # unconditionally (not gated by _skip_prefixes below) — it's a
2065 # cheap in-memory check/clear, not a DB open, and skipping it for
2066 # e.g. static-asset requests would let an idle-but-signature-valid
2067 # session's deadline silently stop advancing/enforcing whenever
2068 # the user's tab is only pulling static assets.
2069 session = scope.get("session")
2070 if session is not None:
2071 _enforce_session_expiry(session)
2072 # Same placement, same reason, different property: expiry is
2073 # "this session has been idle too long", revocation is "this
2074 # session was destroyed server-side (logout, password change)
2075 # but its signed cookie is still replayable". `require_auth`
2076 # covers the latter for the routes that use it; this covers
2077 # the ones that deliberately don't. See the docstring.
2078 _enforce_session_revocation(session)
2080 ctx_tokens = None
2081 # Run the inner app first to let SessionMiddleware populate scope
2082 # Then ensure database is open
2083 if method != "OPTIONS" and not any(
2084 path.startswith(p) for p in self._skip_prefixes
2085 ):
2086 # Session data is in scope["session"] after SessionMiddleware
2087 session_data = scope.get("session", {})
2088 if session_data:
2089 from .dependencies.auth import ensure_user_database
2090 from .dependencies.threadpool import run_db_sync
2092 # Create a minimal request-like object for ensure_user_database
2093 class _MinimalRequest:
2094 def __init__(self, session):
2095 self.session = session
2097 # ensure_user_database opens a SQLCipher connection (PBKDF2
2098 # key derivation, file I/O) which would block the event loop
2099 # for hundreds of ms on first call after login. Offload to a
2100 # threadpool so concurrent requests don't serialize behind it.
2101 #
2102 # Use run_db_sync, not bare asyncio.to_thread: a cold
2103 # open_user_database() call runs the phase-2 rekey path
2104 # (_run_phase2_rekey -> rekey_user_indexes), which opens a
2105 # get_user_db_session() unconditionally, before its own
2106 # marker check (vector_stores/legacy_rekey.py). That leaves a
2107 # thread-local session on whichever asyncio default-executor
2108 # thread served this call, with no cleanup boundary --
2109 # asyncio.to_thread's executor is not drained by
2110 # DatabaseMiddleware (event-loop thread only) or by any
2111 # per-worker cleanup, only ever replaced when that pooled
2112 # thread later serves a different username. run_db_sync
2113 # exists for exactly this: it runs the sync call on the same
2114 # default executor, then cleans up the thread-local DB
2115 # session (and other ambient thread-local state) on that
2116 # worker before returning.
2117 await run_db_sync(
2118 ensure_user_database, _MinimalRequest(session_data)
2119 )
2121 # Publish username/session_id to a contextvar so service-layer
2122 # code that historically read flask_session can fall back
2123 # without requiring a Flask request context.
2124 ctx_tokens = set_request_user(
2125 session_data.get("username"),
2126 session_data.get("session_id"),
2127 )
2129 async def send_wrapper(message):
2130 if message["type"] == "http.response.start":
2131 # Regression #1: (re)stamp the non-remember-me deadline on
2132 # the way out, after the route handler (including a fresh
2133 # login) has had a chance to set username/_remember_me.
2134 # See _stamp_session_expiry's docstring.
2135 current_session = scope.get("session")
2136 if current_session is not None:
2137 _stamp_session_expiry(current_session)
2138 await send(message)
2140 try:
2141 await self.app(scope, receive, send_wrapper)
2142 finally:
2143 if ctx_tokens is not None:
2144 reset_request_user(ctx_tokens)
2146 # Cleanup after request.
2147 #
2148 # Keep the two sweeps independent so one failure cannot skip the
2149 # other. This async middleware can clean event-loop-owned state;
2150 # WorkerCleanupAPIRoute already reclaimed a synchronous
2151 # endpoint's AnyIO-worker state before control returned here.
2152 # Pinned by tests/web/test_teardown_cleanup_asgi.py and
2153 # tests/web/test_db_session_lifecycle_asgi.py.
2154 try:
2155 from ..database.thread_local_session import (
2156 cleanup_dead_threads,
2157 )
2159 cleanup_dead_threads()
2160 except Exception:
2161 logger.debug("Error during post-request dead-thread cleanup")
2163 try:
2164 from ..database.thread_local_session import (
2165 cleanup_current_thread,
2166 )
2168 cleanup_current_thread()
2169 except Exception:
2170 logger.debug("Error during post-request current-thread cleanup")
2173# ---------------------------------------------------------------------------
2174# Request-arrival/duration forensics (CI/TESTING only) — issue #4431
2175# ---------------------------------------------------------------------------
2178class RequestTimingASGIMiddleware:
2179 """Pure ASGI port of ``web/utils/request_timing.RequestTimingMiddleware``.
2181 Named ``...ASGIMiddleware`` rather than reusing the original's name on
2182 purpose: both classes live in the same package, and one of them is a
2183 WSGI callable that would fail silently and confusingly if handed to
2184 ``add_middleware``.
2186 That module is a plain WSGI callable (``__call__(environ,
2187 start_response)``) — main installed it by wrapping ``app.wsgi_app`` in
2188 ``app_factory.create_app`` under CI/TESTING (#4536), and the FastAPI
2189 migration deleted ``app_factory.py`` and lost the wiring with it (#5959).
2190 A WSGI callable cannot be handed to ``app.add_middleware``, so the
2191 transport half is re-expressed here while the *policy* half — the log
2192 format the CI log-grep keys on, the slow-completion threshold, and the
2193 faulthandler freeze dead-man's switch — is imported from the original
2194 module so there is exactly one definition of each.
2196 Pure ASGI rather than ``BaseHTTPMiddleware`` for the same reason as
2197 ``BodySizeLimitMiddleware`` / ``_PathScopedCORSMiddleware``: this sits
2198 outside the streaming SSE routes and ``BaseHTTPMiddleware`` buffers
2199 ``StreamingResponse`` bodies.
2201 One deliberate difference from the WSGI original: the duration here
2202 covers the whole downstream ASGI call, i.e. arrival through the final
2203 response body — which is what #4431 actually wants to measure — rather
2204 than WSGI view execution with response streaming excluded. The arrival
2205 line, whose *absence* during a navigation timeout is the real signal,
2206 is byte-identical.
2207 """
2209 def __init__(self, app):
2210 self.app = app
2211 from .utils.request_timing import _arm_freeze_dump
2213 # Arm the freeze thread-dump dead-man's switch (no-op under pytest).
2214 _arm_freeze_dump()
2216 async def __call__(self, scope, receive, send):
2217 if scope["type"] != "http":
2218 # CORS-style scoping: the forensics are about HTTP request
2219 # arrival. lifespan/websocket scopes pass straight through.
2220 await self.app(scope, receive, send)
2221 return
2223 from .utils.request_timing import (
2224 SLOW_REQUEST_SECONDS,
2225 _arm_freeze_dump,
2226 )
2228 # Re-arm the dead-man's switch: as long as requests keep arriving the
2229 # dump never fires; a freeze (no arrivals) lets it fire and capture
2230 # the stuck thread stacks.
2231 _arm_freeze_dump()
2233 method = scope.get("method", "-")
2234 path = scope.get("path", "-")
2235 # engine.io transport/sid make poll churn correlatable. (sid is
2236 # logged on purpose for correlation; logs are CI-only artifacts.)
2237 if path.startswith("/socket.io"):
2238 query = scope.get("query_string", b"").decode("latin-1", "replace")
2239 path = f"{path}?{query}" if query else path
2240 # Strip CR/LF so a crafted path/query cannot inject fake log lines
2241 # (the forensics output is grep'd downstream).
2242 path = path.replace("\r", "\\r").replace("\n", "\\n")
2243 logger.info(f"[req] > {method} {path}")
2244 start = time.monotonic()
2245 try:
2246 await self.app(scope, receive, send)
2247 finally:
2248 elapsed = time.monotonic() - start
2249 if elapsed >= SLOW_REQUEST_SECONDS:
2250 logger.warning(f"[req] < {method} {path} {elapsed:.1f}s SLOW")
2251 else:
2252 logger.info(f"[req] < {method} {path} {elapsed:.2f}s")
2255def _request_timing_enabled() -> bool:
2256 """Whether to install :class:`RequestTimingASGIMiddleware` on this process.
2258 main's gate was ``os.environ.get("CI") or os.environ.get("TESTING")``,
2259 kept verbatim, plus one addition: never inside a pytest process.
2261 The diagnostic exists for the long-running UI-shard server, which CI
2262 starts as its own process (``python -m local_deep_research.web.app``)
2263 with ``CI`` set — pytest is not imported there, so the gate still arms
2264 exactly where #4431 needs it. The pytest exclusion mirrors
2265 ``request_timing._should_arm_freeze_dump``'s own reasoning (that module
2266 already refuses to arm the faulthandler timer under pytest) and keeps
2267 the in-process middleware stack that
2268 ``tests/web/test_middleware_order_and_headers.py`` pins byte-identical
2269 on a CI runner, where ``CI=true`` is set for the unit-test job too.
2270 """
2271 if "pytest" in sys.modules:
2272 return False
2273 return bool(os.environ.get("CI") or os.environ.get("TESTING"))
2276# ---------------------------------------------------------------------------
2277# Rate limiting
2278# ---------------------------------------------------------------------------
2281def _setup_rate_limiting(app: FastAPI) -> None:
2282 """Configure slowapi rate limiting on the app."""
2283 from slowapi.errors import RateLimitExceeded
2284 from slowapi.middleware import SlowAPIMiddleware
2286 from ..security.log_sanitizer import sanitize_for_log
2287 from .dependencies.rate_limit import _get_client_ip, limiter
2289 app.state.limiter = limiter
2291 # Deliberately `def`, not `async def`. slowapi's SlowAPIMiddleware handles
2292 # the global default limits through `sync_check_limits`, which cannot await
2293 # and therefore DISCARDS an async handler outright:
2294 #
2295 # # cannot execute asynchronous code in a synchronous middleware,
2296 # # -> fallback on default exception handler
2297 # if inspect.iscoroutinefunction(exception_handler):
2298 # exception_handler = _rate_limit_exceeded_handler
2299 # (slowapi/middleware.py)
2300 #
2301 # While this was `async def`, every 429 raised by the middleware path — i.e.
2302 # every undecorated route hitting the global default — silently used
2303 # slowapi's stock handler instead of this one, losing the audit warning
2304 # below, the Retry-After headers, and main's {"error", "message"} body
2305 # contract. Only routes carrying an explicit @limiter.limit decorator
2306 # (the async path) ever reached this function. Nothing here awaits, so
2307 # making it sync costs nothing and covers both paths.
2308 def _rate_limit_exceeded(request: Request, exc: RateLimitExceeded):
2309 # Audit logging for security monitoring — port of main's 429
2310 # errorhandler. _get_client_ip resolves the real IP behind
2311 # trusted proxies (raw request.client would log the proxy).
2312 # Every field is client-influenced, so each goes through
2313 # sanitize_for_log (control, format and line-break characters
2314 # stripped, length capped). The line is space-delimited
2315 # ``field=value``, so the free-text fields are also JSON-quoted:
2316 # json.dumps always double-quotes and escapes ``"`` and ``\``, so a
2317 # key=value reader cannot find a field boundary inside the value
2318 # (repr() would switch to single quotes for a value containing
2319 # ``"``, which such a reader does not honour). The ip value is
2320 # always a validated address or a fixed token (_get_client_ip), so
2321 # it stays unquoted.
2322 path = sanitize_for_log(request.url.path, max_length=200)
2323 user_agent = sanitize_for_log(
2324 request.headers.get("User-Agent", "unknown"), max_length=200
2325 )
2326 logger.warning(
2327 "Rate limit exceeded: "
2328 f"endpoint={json.dumps(path)} "
2329 f"ip={sanitize_for_log(_get_client_ip(request), max_length=64)} "
2330 f"user_agent={json.dumps(user_agent)}"
2331 )
2332 response = JSONResponse(
2333 {
2334 "error": "Too many requests",
2335 "message": "Too many attempts. Please try again later.",
2336 },
2337 status_code=429,
2338 # Stamp the security headers here, the same way the catch-all 500
2339 # handler does. `_setup_rate_limiting` runs AFTER
2340 # `add_middleware(SecurityHeadersMiddleware)`, and Starlette's
2341 # add_middleware is LIFO, so SlowAPIMiddleware ends up OUTSIDE
2342 # SecurityHeadersMiddleware — a 429 it generates never passes
2343 # through it and would otherwise ship with no CSP, no nosniff, no
2344 # X-Frame-Options and no Cache-Control. Reordering the stack would
2345 # be the alternative, but the order is deliberately pinned by
2346 # tests/web/test_middleware_order_and_headers.py.
2347 headers={
2348 name.decode(): value.decode()
2349 for name, value in SecurityHeadersMiddleware.response_headers(
2350 path=request.url.path,
2351 scheme=request.url.scheme,
2352 status=429,
2353 )
2354 },
2355 )
2356 # Restore Retry-After / X-RateLimit-* on 429s. `headers_enabled=True`
2357 # on the Limiter (dependencies/rate_limit.py) alone does not add
2358 # them — slowapi only injects headers when a caller invokes
2359 # `_inject_headers` itself, which its own default
2360 # `_rate_limit_exceeded_handler` (slowapi/extension.py) does; this
2361 # app registers a custom handler instead (for the audit log line
2362 # above and main's ``{"error", "message"}`` body contract) and,
2363 # before this fix, never called it, so live 429s carried none of
2364 # these headers even with the flag on. Flask-Limiter added them on
2365 # main via its own ``after_request`` hook — no direct FastAPI
2366 # equivalent exists, so it has to happen here, per-handler.
2367 #
2368 # Uses the `limiter` closed over above (same object as
2369 # `app.state.limiter`) rather than going through
2370 # `request.app.state.limiter` — the two are identical for the real
2371 # app, but the latter forces every caller of this handler to have
2372 # a real `request.app`, and tests/web/test_rate_limit_coverage.py
2373 # ::TestRateLimitExceededHandler calls this handler directly with
2374 # a bare ``Mock()`` request (pinning the audit-log/body contract in
2375 # isolation, unrelated to this fix). Going through `request.app`
2376 # there silently replaces `response` with an auto-generated Mock
2377 # instead of exercising real header-injection logic.
2378 #
2379 # `request.state.view_rate_limit` is set by slowapi's
2380 # `Limiter.__evaluate_limits` (private method) immediately before
2381 # it raises `RateLimitExceeded` — verified against the installed
2382 # slowapi source (`.venv/lib/*/site-packages/slowapi/__init__.py`)
2383 # rather than assumed, since both the attribute and
2384 # `_inject_headers` are private slowapi API that could rename
2385 # across versions. The whole call is wrapped in a try/except
2386 # (rather than trusting `_inject_headers`'s own internal
2387 # swallow_errors — this Limiter is built with
2388 # `swallow_errors=False`, i.e. it re-raises) so nothing about this
2389 # best-effort header injection — a missing `view_rate_limit`, a
2390 # storage hiccup, or (as in the Mock-request test above) a
2391 # request.state that doesn't behave like a real one — can ever
2392 # turn a 429 into a 500. On any failure `response` simply keeps
2393 # its original value, assigned before the `try`.
2394 #
2395 # tests/web/test_rate_limit_headers_on_429.py pins this against a
2396 # real 429 from a real rate-limited route so a future slowapi
2397 # rename of either private symbol fails loudly here instead of
2398 # silently dropping the headers again.
2399 # Set the headers directly rather than calling
2400 # `limiter._inject_headers`. That method is gated on the Limiter's
2401 # `_headers_enabled`, and this app deliberately leaves that flag OFF
2402 # — turning it on makes slowapi wrap EVERY rate-limited route and
2403 # raise on any handler that returns a plain dict instead of a
2404 # Response, turning ordinary 200s into 500s (see the comment on
2405 # `_limiter_kwargs` in dependencies/rate_limit.py). So calling
2406 # `_inject_headers` here would silently no-op; the values have to be
2407 # computed here instead.
2408 try:
2409 current_limit = getattr(request.state, "view_rate_limit", None)
2410 if current_limit is not None:
2411 item, key_parts = current_limit
2412 reset_at, remaining = limiter.limiter.get_window_stats(
2413 item, *key_parts
2414 )
2415 response.headers["Retry-After"] = str(
2416 max(0, int(reset_at - time.time()))
2417 )
2418 response.headers["X-RateLimit-Limit"] = str(item.amount)
2419 response.headers["X-RateLimit-Remaining"] = str(remaining)
2420 response.headers["X-RateLimit-Reset"] = str(int(reset_at))
2421 except Exception:
2422 logger.debug(
2423 "Could not attach Retry-After/X-RateLimit-* headers to the "
2424 "429 response; returning it without them."
2425 )
2426 return response
2428 app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded)
2430 # Enforce the global DEFAULT_RATE_LIMIT on every route without an
2431 # explicit limit decorator (decorated routes are exempt here — their
2432 # decorators handle their own checks; mounts like /ws have no route
2433 # handler and are skipped). Flask-Limiter's default_limits behaved
2434 # the same way on main.
2435 app.add_middleware(SlowAPIMiddleware)
2438# ---------------------------------------------------------------------------
2439# Static file serving
2440# ---------------------------------------------------------------------------
2443def _add_static_routes(app: FastAPI) -> None:
2444 """Add static file serving routes with cache control."""
2445 from .static_files import static_file_response
2447 static_dir = Path(STATIC_DIR)
2449 # Static assets must not share the API's default per-IP bucket: base.html
2450 # alone pulls ~29 assets, served must-revalidate, so a few page loads
2451 # could exhaust a user's whole hourly budget. main exempted these.
2452 from .dependencies.rate_limit import limiter as _limiter
2454 @app.get("/favicon.ico", include_in_schema=False)
2455 @_limiter.exempt
2456 async def favicon():
2457 from fastapi.responses import FileResponse
2459 favicon_path = static_dir / "favicon.ico"
2460 if favicon_path.exists():
2461 return FileResponse(str(favicon_path), media_type="image/x-icon")
2462 return JSONResponse({"error": "Not found"}, status_code=404)
2464 @app.get("/static/{path:path}", include_in_schema=False)
2465 @_limiter.exempt
2466 async def serve_static(path: str):
2467 response = static_file_response(path)
2468 if response is not None:
2469 return response
2470 return JSONResponse({"error": "Not found"}, status_code=404)
2473# ---------------------------------------------------------------------------
2474# Template helpers (replaces Flask's context_processor / jinja_env.globals)
2475# ---------------------------------------------------------------------------
2477# Matches url_for('name') / url_for("name") usages in templates. Shared by
2478# _validate_url_for_bindings (below) and
2479# tests/web/templates/test_url_for_links.py, which independently extracts
2480# the same names to fence dead links.
2481_URL_FOR_NAME_RE = re.compile(r"url_for\(\s*['\"]([^'\"]+)['\"]")
2484def _setup_template_globals() -> None:
2485 """Register template globals for Jinja2 (CSRF, url_for, flash, vite, themes, etc.)."""
2486 env = templates.env
2488 # Vite asset helpers (must be registered at module level, not in lifespan)
2489 from .utils.vite_helper import vite
2491 vite.init_for_fastapi(STATIC_DIR, templates)
2493 # CSRF token — needs request, so we provide a callable that reads from
2494 # the template context (where request is passed per-render).
2495 # Templates call {{ csrf_token() }} which reads _csrf_request from context.
2497 def csrf_token_factory():
2498 """Returns empty string by default. Overridden per-request via context."""
2499 return ""
2501 env.globals["csrf_token"] = csrf_token_factory
2503 # Flash messages — same pattern: needs request.
2504 env.globals["get_flashed_messages"] = lambda with_categories=False: []
2506 # Theme helpers
2507 from .themes import get_theme_metadata, get_themes, get_themes_json
2509 env.globals["get_themes"] = get_themes
2510 env.globals["get_themes_json"] = get_themes_json
2511 env.globals["get_theme_metadata"] = get_theme_metadata
2513 # Session — default empty dict, overridden per-request via context
2514 env.globals["session"] = {}
2516 # Egress scope — default to the registered fallback so templates
2517 # rendered WITHOUT render_template() (e.g. the index route's direct
2518 # TemplateResponse) can still reference egress_scope on
2519 # <body data-scope=…> without raising an UndefinedError. render_template()
2520 # overrides this per-render with the user's resolved scope.
2521 from ..security.egress.policy import DEFAULT_EGRESS_SCOPE
2523 env.globals["egress_scope"] = DEFAULT_EGRESS_SCOPE
2525 # Kept INSIDE this function as a single direct dict literal, not hoisted
2526 # to module scope: .pre-commit-hooks/check-url-for-targets.py parses this
2527 # function statically and requires exactly one literal map here ("Expected
2528 # one direct literal map"). _validate_url_for_bindings does not need the
2529 # dict — it resolves names through templates.env.globals["url_for"], the
2530 # same closure the templates use.
2531 # url_for wrapper map (Flask-style blueprint.endpoint -> path).
2532 _URL_MAP = {
2533 # Auth
2534 "auth.login": "/auth/login",
2535 "auth.login_page": "/auth/login",
2536 "auth.register": "/auth/register",
2537 "auth.register_page": "/auth/register",
2538 "auth.logout": "/auth/logout",
2539 "auth.change_password": "/auth/change-password",
2540 "auth.change_password_page": "/auth/change-password",
2541 # Pages
2542 "index": "/",
2543 "static": "/static",
2544 "history.history_page": "/history/",
2545 "chat.chat_page": "/chat/",
2546 # Library / RAG
2547 "library.library_page": "/library/",
2548 "library.download_manager_page": "/library/download-manager",
2549 "rag.collections_page": "/library/collections",
2550 "rag.embedding_settings_page": "/library/embedding-settings",
2551 # Zotero page lives under the /library prefix; without this the
2552 # dot-to-slash fallback yields a dead /zotero/zotero_page link.
2553 "zotero.zotero_page": "/library/zotero",
2554 # Notes + unified-search pages (the dot-to-slash fallback would
2555 # otherwise produce /notes/notes_page and
2556 # /unified_search/unified_search_page dead links).
2557 "notes.notes_page": "/notes/",
2558 "unified_search.unified_search_page": "/library/search/",
2559 # News
2560 "news.news_page": "/news/",
2561 "news.subscriptions_page": "/news/subscriptions",
2562 # Metrics
2563 "metrics.metrics_dashboard": "/metrics/",
2564 "metrics.journal_quality": "/metrics/journals",
2565 # Benchmark
2566 "benchmark.index": "/benchmark/",
2567 # Currently also produced by the dot-to-slash fallback, but pin it
2568 # so a rename of the route function can't silently break the
2569 # sidebar link.
2570 "benchmark.results": "/benchmark/results",
2571 # Settings (page + form action)
2572 "settings.settings_page": "/settings/",
2573 "settings.save_settings": "/settings/save_settings",
2574 "settings.save_all_settings": "/settings/save_all_settings",
2575 "settings.main_config_page": "/settings/main",
2576 "settings.collections_config_page": "/settings/collections",
2577 "settings.api_keys_config_page": "/settings/api_keys",
2578 "settings.llm_config_page": "/settings/llm",
2579 "settings.search_engines_config_page": "/settings/search_engines",
2580 }
2582 def url_for(name, **kwargs):
2583 path = _URL_MAP.get(name, f"/{name.replace('.', '/')}")
2584 if name == "static" and "filename" in kwargs:
2585 path = f"/static/{kwargs['filename']}"
2586 elif name == "static" and "path" in kwargs: 2586 ↛ 2587line 2586 didn't jump to line 2587 because the condition on line 2586 was never true
2587 path = f"/static/{kwargs['path']}"
2588 # Append query params
2589 query_params = {
2590 k: v for k, v in kwargs.items() if k not in ("filename", "path")
2591 }
2592 if query_params:
2593 from urllib.parse import urlencode
2595 path += "?" + urlencode(query_params)
2596 return path
2598 env.globals["url_for"] = url_for
2601# ---------------------------------------------------------------------------
2602# Create the FastAPI application
2603# ---------------------------------------------------------------------------
2605# Deliberately does NOT check the generic `CI` / `TESTING` variables, which an
2606# earlier version did. This flag suppresses the `Secure` attribute on session
2607# cookies (SecureCookieMiddleware) *and* both operator warnings about serving
2608# a public instance over plain HTTP, so any CD pipeline or PaaS that exports
2609# CI=true would have shipped session cookies without `Secure` over HTTPS, and
2610# silenced the warning that would have said so.
2611#
2612# This is the same reasoning already written down in
2613# database/sqlcipher_utils.py::_get_min_kdf_iterations, which refuses to relax
2614# the KDF floor on CI/TESTING for exactly this reason. The cookie path had not
2615# been given that treatment.
2616#
2617# Only the explicit operator switch controls cookie test mode. Pytest's
2618# per-test marker changes between collection and execution, so consulting it
2619# here would make cookie protection and docs exposure depend on import order.
2620# Keep the production cookie behavior by default, including under pytest.
2621# LDR_TEST_MODE is parsed as a boolean, so "0" and "false" do not enable it.
2622is_testing = to_bool(os.getenv("LDR_TEST_MODE"))
2624# Public API docs are disabled by default — a multi-user deployment
2625# shouldn't expose the full schema unauthenticated. Set LDR_EXPOSE_DOCS=true
2626# to re-enable (useful for internal/dev instances).
2627_expose_docs = os.getenv("LDR_EXPOSE_DOCS", "").lower() in ("true", "1", "yes")
2628_docs_enabled = _expose_docs and not is_testing
2630app = FastAPI(
2631 title="Local Deep Research",
2632 version=__version__,
2633 docs_url="/api/docs" if _docs_enabled else None,
2634 # Gate the OpenAPI schema endpoint together with the Swagger UI:
2635 # /openapi.json IS the full API schema, so leaving it on its default
2636 # "/openapi.json" while disabling docs_url still exposes the entire
2637 # surface unauthenticated. app.openapi() remains callable in-process.
2638 openapi_url="/openapi.json" if _docs_enabled else None,
2639 redoc_url=None,
2640 lifespan=lifespan,
2641)
2643# FastAPI runs plain ``def`` endpoints on AnyIO workers, whereas ASGI
2644# middleware teardown runs on the event-loop thread. Install an endpoint
2645# wrapper so thread-local DB sessions are released on the worker that owns
2646# them. Populated module routers receive the same dispatch wrapper in
2647# ``_mount_all`` without replacing their registered endpoint objects.
2648from .dependencies.threadpool import WorkerCleanupAPIRoute # noqa: E402
2650app.router.route_class = WorkerCleanupAPIRoute
2652# ASGI middleware stack.
2653# Starlette add_middleware is LIFO: last added = outermost.
2654# Desired order (outer→inner): SecureCookie → SecurityHeaders → Session → CSRF → Database → app
2655#
2656# So we add in reverse: Database first, then CSRF, then Session, etc.
2658app.add_middleware(DatabaseMiddleware)
2660# CSRF runs after Session populates scope["session"] and rejects state-changing
2661# requests with a missing/invalid X-CSRFToken header before they reach handlers.
2662from .dependencies.csrf import CSRFMiddleware # noqa: E402
2664app.add_middleware(CSRFMiddleware)
2666app.add_middleware(
2667 SessionMiddleware,
2668 secret_key=SECRET_KEY,
2669 session_cookie="session",
2670 max_age=get_security_default("security.session_remember_me_days", 30)
2671 * 24
2672 * 3600,
2673 # Include the login cookie on safe top-level navigation from other sites.
2674 # Lax still withholds it from cross-site unsafe requests, and
2675 # CSRFMiddleware requires the session token on unsafe methods. But Lax
2676 # lets another site's link open any page signed in, so GET routes must
2677 # not change state (indexing and model discovery use POST), and a
2678 # page-load script must not turn URL parameters into a state-changing
2679 # request, which would carry the page's own CSRF token. chat.js only
2680 # pre-fills `?q=` into a chat without messages, without focus, and never
2681 # sends it by itself; the chat page also redirects `?q=` away from links
2682 # that Sec-Fetch-Site marks as coming from another origin
2683 # (web/routers/chat.py), and the login page drops it from a chat `next`
2684 # (web/routers/auth.py). The main form submits its chat message through
2685 # the protected API before opening the chat, without browser staging.
2686 same_site="lax",
2687 https_only=False, # Handled dynamically by SecureCookieMiddleware
2688)
2690# Runs just outside SessionMiddleware so it sees the Set-Cookie
2691# header SessionMiddleware just wrote, and can strip Max-Age/Expires
2692# for non-remember-me logins. See the class docstring.
2693app.add_middleware(RememberMeMiddleware)
2695# Body cap sits inside SecurityHeaders so its 413 responses still get
2696# security headers, but outside Session/CSRF so over-limit bodies are
2697# rejected before anything buffers them.
2698app.add_middleware(BodySizeLimitMiddleware)
2700app.add_middleware(SecurityHeadersMiddleware)
2701app.add_middleware(SecureCookieMiddleware, testing=is_testing)
2704# Prefixes that carried CORS headers on main. `SecurityHeaders._is_api_route`
2705# (security/security_headers.py, deleted by the FastAPI migration) gated CORS to
2706# exactly these three, so an operator who whitelisted an origin for API access
2707# did not thereby grant it cross-origin reads of every HTML page. Restored here.
2708_CORS_API_PREFIXES = ("/api/", "/research/api/", "/history/api")
2711class _PathScopedCORSMiddleware:
2712 """Apply Starlette's CORSMiddleware only to API paths.
2714 Starlette's CORSMiddleware has no path predicate: mounted app-wide it
2715 answers preflights and stamps `Access-Control-Allow-Origin` on every route,
2716 HTML pages included. main scoped that to three prefixes and the migration
2717 dropped the scoping along with `_is_api_route`.
2719 Pure ASGI rather than BaseHTTPMiddleware, deliberately: this sits in front
2720 of the streaming SSE routes, and BaseHTTPMiddleware buffers
2721 StreamingResponse bodies (the defect already tracked for
2722 SlowAPIMiddleware). Non-matching paths are handed to the inner app
2723 untouched, so the pass-through case adds one string comparison and no
2724 wrapping at all.
2726 Non-HTTP scopes (websocket, lifespan) bypass CORS entirely — the CORS
2727 protocol is HTTP-only, and the Socket.IO mount does its own origin check.
2728 """
2730 def __init__(self, app, prefixes, cors_factory):
2731 self.app = app
2732 self.prefixes = tuple(prefixes)
2733 # Build the real CORSMiddleware once, wrapping the inner app, so a
2734 # matching request gets Starlette's full preflight/origin handling.
2735 self.cors_app = cors_factory(app)
2737 async def __call__(self, scope, receive, send):
2738 if scope["type"] != "http":
2739 await self.app(scope, receive, send)
2740 return
2741 path = scope.get("path", "")
2742 if path.startswith(self.prefixes):
2743 await self.cors_app(scope, receive, send)
2744 else:
2745 await self.app(scope, receive, send)
2748def _configure_cors(app: FastAPI) -> None:
2749 """Restore configurable cross-origin access for API clients.
2751 Ports the CORS support main applied in security_headers.py from
2752 ``security.cors.allowed_origins`` (env: LDR_SECURITY_CORS_ALLOWED_ORIGINS),
2753 which the FastAPI migration dropped — the setting was live config on
2754 main but reached no code on this branch. Uses Starlette's
2755 CORSMiddleware (correct preflight/OPTIONS handling, origin matching)
2756 rather than hand-injecting headers.
2758 Fail closed: with the setting empty (the default) no CORS middleware
2759 is registered, so the app stays same-origin only — exactly main's
2760 empty-default behavior. Same-origin requests carry no Origin header
2761 and never receive CORS headers regardless, so this is effectively
2762 scoped to genuine cross-origin API calls. A separator-only value
2763 (e.g. ``","``) parses to no origins at all and is treated the same
2764 way — no middleware registered — rather than as an allowlist that
2765 can never match anything.
2766 """
2767 from ..settings.env_registry import get_env_setting
2769 configured = get_env_setting("security.cors.allowed_origins")
2770 if not configured:
2771 return
2773 if configured.strip() == "*":
2774 # Credentials with a wildcard origin is forbidden by the CORS spec
2775 # (and was a startup error on main). Allow the wildcard for
2776 # non-credentialed access only.
2777 origins = ["*"]
2778 allow_credentials = False
2779 else:
2780 origins = [o.strip() for o in configured.split(",") if o.strip()]
2781 allow_credentials = False
2782 if not origins:
2783 # Every entry was empty (e.g. "," or " , "): the same intent
2784 # as unset. Registering middleware here would produce an
2785 # allowlist that can never match any Origin and a "CORS
2786 # enabled for API origins: []" log that misleadingly claims
2787 # CORS is on, so treat it exactly like the unset case above.
2788 return
2790 from starlette.middleware.cors import CORSMiddleware
2792 app.add_middleware(
2793 _PathScopedCORSMiddleware,
2794 prefixes=_CORS_API_PREFIXES,
2795 cors_factory=lambda inner: CORSMiddleware(
2796 inner,
2797 allow_origins=origins,
2798 allow_credentials=allow_credentials,
2799 allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
2800 allow_headers=[
2801 "Content-Type",
2802 "Authorization",
2803 "X-Requested-With",
2804 "X-HTTP-Method-Override",
2805 "X-CSRFToken",
2806 ],
2807 max_age=3600,
2808 ),
2809 )
2810 logger.info(
2811 f"CORS enabled for API origins: {origins} "
2812 f"(scoped to {', '.join(_CORS_API_PREFIXES)})"
2813 )
2816# Rate limiting via slowapi + exception handlers
2817_setup_rate_limiting(app)
2818_register_exception_handlers(app)
2820# Registered last of the request-handling middleware, so it is the outermost
2821# of them: a preflight OPTIONS is answered before auth, CSRF, or rate limiting
2822# ever run. (The CI-only request-timing layer below is added after it and so
2823# sits further out still, but it only logs and passes every scope through, so
2824# it changes nothing about that ordering.)
2825_configure_cors(app)
2827# CI/test-only request forensics for the #4431 navigation-stall hunts: logs
2828# every request's arrival + duration so a silent log window can be attributed
2829# to "request never arrived" vs "request hung in the app". Ports #4536, which
2830# merge 5ad5f5a1b reverted along with app_factory.py (#5959). Added LAST of
2831# all `add_middleware` calls because that list is LIFO — last added is
2832# outermost — which is the only position from which it measures true
2833# arrival-to-response time.
2834if _request_timing_enabled(): 2834 ↛ 2835line 2834 didn't jump to line 2835 because the condition on line 2834 was never true
2835 app.add_middleware(RequestTimingASGIMiddleware)
2836 logger.info("Request-timing forensics middleware enabled (CI/TESTING)")
2838# Static file routes
2839_add_static_routes(app)
2841# Template globals
2842_setup_template_globals()
2844# ---------------------------------------------------------------------------
2845# Mount routers and Socket.IO (deferred imports to avoid E402)
2846# ---------------------------------------------------------------------------
2849def _mount_all(app: FastAPI) -> None:
2850 """Mount all routers and Socket.IO on the app."""
2851 from ..database.session_context import (
2852 DatabaseSessionUnavailableError,
2853 get_user_db_session,
2854 )
2855 from ..settings.logger import log_settings
2856 from ..utilities.db_utils import get_settings_manager
2857 from .dependencies.threadpool import prepare_router_for_worker_cleanup
2858 from .routers.api_v1 import router as api_v1_router
2859 from .services.socketio_asgi import socket_app
2861 # Mount Socket.IO
2862 app.mount("/ws", socket_app)
2864 # Include all routers — error-tolerant loading during migration.
2865 # Routers with unresolved import issues will be skipped with a warning.
2866 _routers = {
2867 "api_v1": api_v1_router,
2868 }
2870 # Phase 2+ routers — Phase 2-7 is complete; an import error here is a
2871 # production-class regression, not a migration-in-progress signal. Fail
2872 # loud at startup instead of silently serving 404 for the whole router.
2873 _router_modules = [
2874 ("auth", ".routers.auth"),
2875 ("research", ".routers.research"),
2876 ("history", ".routers.history"),
2877 ("settings", ".routers.settings"),
2878 ("metrics", ".routers.metrics"),
2879 ("api", ".routers.api"),
2880 ("context_overflow", ".routers.context_overflow_api"),
2881 ("news_api", ".routers.news_flask_api"),
2882 ("news_pages", ".routers.news_pages"),
2883 ("benchmark", ".routers.benchmark"),
2884 ("followup", ".routers.followup"),
2885 ("library", ".routers.library"),
2886 ("rag", ".routers.rag"),
2887 ("library_delete", ".routers.library_delete"),
2888 ("library_search", ".routers.library_search"),
2889 ("zotero", ".routers.zotero"),
2890 ("notes", ".routers.notes"),
2891 ("unified_search", ".routers.unified_search"),
2892 ("scheduler", ".routers.scheduler"),
2893 ("chat", ".routers.chat"),
2894 ]
2896 import importlib
2898 for name, module_path in _router_modules:
2899 mod = importlib.import_module(module_path, package=__package__)
2900 if not hasattr(mod, "router"):
2901 raise RuntimeError(
2902 f"Router module '{module_path}' has no 'router' attribute"
2903 )
2904 _routers[name] = mod.router
2906 for name, r in _routers.items():
2907 app.include_router(r)
2909 # include_router() preserves the source APIRoute class rather than using
2910 # app.router.route_class for already-populated module routers. Apply the
2911 # cleanup to the copied dispatch targets after every router is mounted;
2912 # route.endpoint remains the original handler for introspection, SlowAPI,
2913 # and route-identity security checks.
2914 prepare_router_for_worker_cleanup(app.router)
2916 @app.get("/", include_in_schema=False)
2917 def index(request: Request):
2918 """Root route - redirect to login if not authenticated."""
2919 if not request.session.get("username"):
2920 return RedirectResponse(url="/auth/login", status_code=302)
2922 # Same gate as every protected route. Reading the username from the
2923 # cookie alone would let a session that lost its own credential render
2924 # the user's saved settings: without a session-scoped password,
2925 # get_user_db_session falls back to a sibling session's password.
2926 # require_auth rejects that session (and revokes it) instead.
2927 from .dependencies.auth import (
2928 database_unavailable_exception,
2929 end_rejected_session,
2930 require_auth,
2931 )
2933 try:
2934 username = require_auth(request)
2935 except HTTPException as exc:
2936 if exc.status_code == 503:
2937 # The retryable 503 of a database that cannot be checked
2938 # right now: for a browser, the branded page that retries
2939 # here; for a JSON caller, or if it cannot render, the
2940 # 503 itself. The session is kept either way.
2941 page = _database_unavailable_page(request, exc)
2942 if page is None:
2943 raise
2944 return page
2945 if exc.status_code != 401: 2945 ↛ 2946line 2945 didn't jump to line 2946 because the condition on line 2945 was never true
2946 raise
2947 # One require_auth 401 keeps the cookie (database not connected
2948 # but still recoverable). /auth/login sends any cookie with a
2949 # username back here, so redirecting without ending the session
2950 # would loop. Every protected page's 401 funnels through here.
2951 end_rejected_session(request)
2952 return RedirectResponse(url="/auth/login", status_code=302)
2954 from ..constants import get_available_strategies
2955 from ..security.egress.policy import (
2956 DEFAULT_EGRESS_SCOPE,
2957 effective_scope_for_display,
2958 unprotected_egress_allowed,
2959 )
2960 from ..settings.exceptions import SettingsValueError
2961 from ..settings.manager import check_env_setting
2963 settings = {}
2964 try:
2965 with get_user_db_session(
2966 username, session_id=request.session.get("session_id")
2967 ) as db_session:
2968 if db_session: 2968 ↛ 3082line 2968 didn't jump to line 3082
2969 sm = get_settings_manager(db_session, username)
2970 # Operator-gated escape hatch (#5148 / 87537d9ec): the
2971 # raw stored scope may be a legacy/invalid value or an
2972 # "unprotected" selection the operator has since
2973 # disabled — normalise what we DISPLAY through the same
2974 # helper the policy engine uses so the UI never shows a
2975 # scope the run-time layer would reject. allow_
2976 # unprotected_egress gates the <option value="unprotected">
2977 # entry in research.html's select; the *_env_locked flags
2978 # let it grey out controls the operator pinned via env
2979 # vars (server-side overrides are enforced independently
2980 # in routers/research.py::_apply_policy_overrides).
2981 raw_egress_scope = sm.get_setting(
2982 "policy.egress_scope", DEFAULT_EGRESS_SCOPE
2983 )
2984 effective_egress_scope = effective_scope_for_display(
2985 raw_egress_scope
2986 )
2987 settings = {
2988 "llm_provider": sm.get_setting(
2989 "llm.provider", "ollama"
2990 ),
2991 "llm_model": sm.get_setting("llm.model", ""),
2992 "llm_openai_endpoint_url": sm.get_setting(
2993 "llm.openai_endpoint.url", ""
2994 ),
2995 "llm_ollama_url": sm.get_setting("llm.ollama.url"),
2996 "llm_lmstudio_url": sm.get_setting("llm.lmstudio.url"),
2997 "llm_local_context_window_size": sm.get_setting(
2998 "llm.local_context_window_size"
2999 ),
3000 "search_tool": sm.get_setting("search.tool", ""),
3001 "search_iterations": sm.get_setting(
3002 "search.iterations", 3
3003 ),
3004 "search_questions_per_iteration": sm.get_setting(
3005 "search.questions_per_iteration", 2
3006 ),
3007 "search_strategy": sm.get_setting(
3008 "search.search_strategy", "source-based"
3009 ),
3010 # Egress-policy controls (Stage 1c UI) — main loads
3011 # these so the research page reflects the saved policy
3012 # instead of silently defaulting (fail-open).
3013 "policy_egress_scope": effective_egress_scope,
3014 "egress_scope_env_locked": check_env_setting(
3015 "policy.egress_scope"
3016 )
3017 is not None,
3018 "allow_unprotected_egress": unprotected_egress_allowed(),
3019 "llm_local_env_locked": check_env_setting(
3020 "llm.require_local_endpoint"
3021 )
3022 is not None,
3023 "embeddings_local_env_locked": check_env_setting(
3024 "embeddings.require_local"
3025 )
3026 is not None,
3027 "llm_require_local_endpoint": sm.get_setting(
3028 "llm.require_local_endpoint", False
3029 ),
3030 "embeddings_require_local": sm.get_setting(
3031 "embeddings.require_local", False
3032 ),
3033 }
3034 except DatabaseSessionUnavailableError:
3035 # require_auth's probe succeeded, but opening the session for the
3036 # settings read then met a rekey sidecar or epoch fault: an
3037 # fd-exhaustion spike can clear and come back between the two
3038 # probes. It says nothing about this session, so answer
3039 # require_auth's own retryable 503 and keep the cookie, the
3040 # server-side session and the stored password, rather than the
3041 # logout below (a browser gets it as the branded page that
3042 # retries here). get_user_db_session already logged the fault's
3043 # type name.
3044 logger.warning(
3045 "Root route could not open the database for user {}; "
3046 "answering 503 and keeping the session",
3047 username,
3048 )
3049 unavailable = database_unavailable_exception()
3050 page = _database_unavailable_page(request, unavailable)
3051 if page is None: 3051 ↛ 3052line 3051 didn't jump to line 3052 because the condition on line 3051 was never true
3052 raise unavailable from None
3053 return page
3054 except SettingsValueError:
3055 # An invalid configured setting is not the password case below.
3056 # Clearing the session would only send the user back here after
3057 # the next login, where the same read fails again: a login
3058 # loop. The app's SettingsValueError handler shows the
3059 # diagnostic instead, and the session is kept.
3060 raise
3061 except Exception:
3062 # Password not available (e.g. server restarted) — force re-login.
3063 #
3064 # Log before redirecting. This handler wraps ~70 lines (session
3065 # acquisition, settings manager, egress scope, and 18 get_setting
3066 # calls) on the app's ROOT route, so it catches far more than the
3067 # documented password case — a bug in any of those reads presents
3068 # to the user as "I keep getting randomly logged out" and left no
3069 # trace at any log level. Flask had no try/except here at all
3070 # (web/app_factory.py's index()), so the same fault produced a
3071 # logged 500; swallowing it silently is a diagnosability
3072 # regression, not a behaviour improvement. The redirect itself is
3073 # correct for the case named above and is kept.
3074 logger.exception(
3075 "Root route failed to load settings for user {} — clearing "
3076 "session and redirecting to login",
3077 request.session.get("username"),
3078 )
3079 request.session.clear()
3080 return RedirectResponse(url="/auth/login", status_code=302)
3082 log_settings(settings, "Research page settings loaded")
3084 return templates.TemplateResponse(
3085 request=request,
3086 name="pages/research.html",
3087 context={
3088 "settings": settings,
3089 "strategies": get_available_strategies(),
3090 },
3091 )
3094_mount_all(app)
3097# ---------------------------------------------------------------------------
3098# Startup validation: every url_for() name used by a template must resolve
3099# to a route actually mounted on the app.
3100# ---------------------------------------------------------------------------
3103def _validate_url_for_bindings(app: "FastAPI") -> None:
3104 """Fail loudly at import/startup time if the Flask-compat ``url_for``
3105 shim would silently hand any template a dead link.
3107 ``url_for``'s fallback (``f"/{name.replace('.', '/')}"``) produces a
3108 plausible-looking but wrong URL for any name missing from ``_URL_MAP``
3109 — that already shipped one dead link (``chat.chat_page`` ->
3110 ``/chat/chat_page``, see ``tests/web/templates/test_url_for_links.py``).
3111 Rather than raise inside ``url_for()`` at render time (which would turn
3112 one cosmetic bad link into a 500 for an otherwise-working page — a worse
3113 failure mode for a self-hosted app), this scans every template for
3114 ``url_for("name")`` usages, resolves each one exactly as the real shim
3115 would, and checks the RESOLVED path against the live route table. This
3116 must run AFTER ``_mount_all(app)`` -- ``app.routes`` is empty at the
3117 point ``_setup_template_globals()`` runs (before routers are mounted),
3118 so validating there would trivially "pass" with an unpopulated table.
3120 Deliberately does NOT restrict itself to `_URL_MAP` keys: the
3121 `chat.chat_page` bug was a MISSING entry falling through to the
3122 dot-to-slash guess, which key-only validation would never have caught.
3124 Matching strategy: a resolved path is valid if EITHER
3125 (a) it exactly equals the ``path`` of some plain (parameter-free)
3126 ``Route`` — with a trailing-slash-tolerant comparison, since pages
3127 are linked both with and without one, or
3128 (b) it falls under the prefix of something that is *designed* to
3129 swallow an arbitrary nested sub-path: an ``app.mount(...)``
3130 (``Mount`` — e.g. the Socket.IO ASGI app at ``/ws``) or a ``Route``
3131 using Starlette's catch-all ``{name:path}`` converter (e.g.
3132 ``serve_static``'s ``/static/{path:path}``, which is exactly how
3133 ``url_for("static", filename=...)`` is validated without a special
3134 case).
3136 Every OTHER parameterized route (e.g. ``/chat/{session_id}``, a plain
3137 string converter bound to one specific, known value) is intentionally
3138 EXCLUDED from matching, even though Starlette's own ``Route.matches()``
3139 would happily match any string there. That was tried first and
3140 rejected: ``/chat/{session_id}`` matches literally any string under
3141 ``/chat/``, including ``/chat/totally-wrong-path`` — the exact shape of
3142 wrong path a missing ``_URL_MAP`` entry produces — so a
3143 ``Route.matches()``-based check would NOT have caught the
3144 ``chat.chat_page`` regression this validator exists to prevent.
3145 """
3146 from starlette.routing import Mount, Route
3148 url_for = templates.env.globals["url_for"]
3150 template_dir = Path(TEMPLATE_DIR)
3151 names: set[str] = set()
3152 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Startup link validation enumerates only the installed package HTML templates
3153 for tpl_path in template_dir.rglob("*.html"):
3154 names.update(
3155 _URL_FOR_NAME_RE.findall(
3156 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Template path comes from the installed package HTML glob above
3157 tpl_path.read_text(encoding="utf-8", errors="replace")
3158 )
3159 )
3161 if not names:
3162 # Deliberately NOT fatal. An empty scan is indistinguishable from a
3163 # packaging shape: TEMPLATE_DIR is computed from
3164 # importlib_resources.as_file(), whose temp directory is removed when
3165 # its context manager exits, so a zip-imported / pex / frozen install
3166 # can legitimately have no readable templates directory here.
3167 # Jinja2Templates itself tolerates that (FileSystemLoader never checks
3168 # the directory exists), so the pre-existing behaviour is "HTML pages
3169 # fail, the API keeps serving". Raising here would escalate that to
3170 # "the process does not import at all", which is strictly worse and is
3171 # exactly the trade this function's own docstring rejects.
3172 logger.warning(
3173 "url_for startup validation found no url_for() usages under "
3174 f"{template_dir}; skipping link validation. If this is a normal "
3175 "source install the templates directory has moved, and HTML "
3176 "rendering is already broken."
3177 )
3178 return
3180 static_paths: set[str] = set()
3181 wildcard_prefixes: set[str] = set()
3182 _catchall_re = re.compile(r"\{[^{}:]+:path\}")
3183 for route in app.routes:
3184 path = getattr(route, "path", None)
3185 if not path: 3185 ↛ 3186line 3185 didn't jump to line 3186 because the condition on line 3185 was never true
3186 continue
3187 if isinstance(route, Mount):
3188 wildcard_prefixes.add(path.rstrip("/"))
3189 elif isinstance(route, Route): 3189 ↛ 3183line 3189 didn't jump to line 3183 because the condition on line 3189 was always true
3190 if "{" not in path:
3191 static_paths.add(path)
3192 static_paths.add(path.rstrip("/") or "/")
3193 elif _catchall_re.search(path):
3194 wildcard_prefixes.add(path.split("{", 1)[0].rstrip("/"))
3195 # else: parameterized on a specific value (e.g. {session_id}) --
3196 # not a wildcard bucket url_for is allowed to land in.
3198 def _under_wildcard_prefix(lookup: str) -> bool:
3199 return any(
3200 lookup == prefix or lookup.startswith(prefix + "/")
3201 for prefix in wildcard_prefixes
3202 )
3204 dead: list[str] = []
3205 for name in sorted(names):
3206 path = (
3207 url_for(name, filename="x") if name == "static" else url_for(name)
3208 )
3209 # Normalise BOTH sides. static_paths stores stripped and
3210 # unstripped forms, but without stripping the lookup the
3211 # tolerance was one-directional: nine _URL_MAP values end in "/"
3212 # and pass only because each of those routers happens to declare
3213 # @router.get("/"). Changing one to @router.get("") — invisible
3214 # at runtime thanks to redirect_slashes — would have bricked boot.
3215 lookup = path.split("?", 1)[0]
3216 lookup = lookup.rstrip("/") or "/"
3217 if lookup in static_paths or _under_wildcard_prefix(lookup):
3218 continue
3219 dead.append(f"{name!r} -> {path!r}")
3221 if dead:
3222 detail = (
3223 "the following template url_for() names resolve to paths that "
3224 "are not mounted on the app. Add/fix an entry in _URL_MAP "
3225 "(web/fastapi_app.py) for each:\n " + "\n ".join(dead)
3226 )
3227 # Strict only when asked. A dead nav link is a cosmetic defect; making
3228 # it un-bootable turns one broken link into total unavailability,
3229 # which is a worse failure mode than the render-time raise this
3230 # function already rejects for the same reason. CI and developers get
3231 # the hard failure via LDR_STRICT_TEMPLATE_LINKS and via
3232 # tests/web/templates/test_url_for_links.py, which asserts the same
3233 # property offline; a running deployment gets a loud log and keeps
3234 # serving.
3235 if os.environ.get("LDR_STRICT_TEMPLATE_LINKS", "").lower() in (
3236 "1",
3237 "true",
3238 "yes",
3239 ):
3240 raise RuntimeError("url_for() startup validation failed: " + detail)
3241 logger.error("url_for() startup validation failed: " + detail)
3244_validate_url_for_bindings(app)