Coverage for src/local_deep_research/web/routers/settings.py: 96%
1575 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Settings Routes Module
4This module handles all settings-related HTTP endpoints for the application.
6CHECKBOX HANDLING PATTERN:
7--------------------------
8This module supports TWO submission modes to handle checkboxes correctly:
10**MODE 1: AJAX/JSON Submission (Primary - /save_all_settings)**
11- JavaScript intercepts form submission with e.preventDefault()
12- Checkbox values read directly from DOM via checkbox.checked
13- Data sent as JSON: {"setting.key": true/false}
14- Hidden fallback inputs are managed but NOT used in this mode
15- Provides better UX with instant feedback and validation
17**MODE 2: Traditional POST Submission (Fallback - /save_settings)**
18- Used when JavaScript is disabled (accessibility/no-JS environments)
19- Browser submits form data naturally via request.form
20- Hidden fallback pattern CRITICAL here:
21 * Checked checkbox: Submits checkbox value, hidden input disabled
22 * Unchecked checkbox: Submits hidden input value "false"
23- Ensures unchecked checkboxes are captured (HTML limitation workaround)
25**Implementation Details:**
261. Each checkbox has `data-hidden-fallback` attribute → hidden input ID
272. checkbox_handler.js manages hidden input disabled state
283. AJAX mode: settings.js reads checkbox.checked directly (lines 2233-2240)
294. POST mode: Flask reads request.form including enabled hidden inputs
305. Both modes use convert_setting_value() for consistent boolean conversion
32**Why Both Patterns?**
33- AJAX: Better UX, immediate validation, no page reload
34- Traditional POST: Accessibility, progressive enhancement, JavaScript-free operation
35- Hidden inputs: Only meaningful for traditional POST, ignored in AJAX mode
37This dual-mode approach ensures the app works for all users while providing
38optimal experience when JavaScript is available.
39"""
41from fastapi import APIRouter, Depends, HTTPException, Request
42from fastapi.responses import JSONResponse, RedirectResponse
43from ..dependencies.auth import (
44 require_auth,
45)
46from ..dependencies.rate_limit import (
47 SETTINGS_RATE_LIMIT,
48 _user_key,
49 limiter,
50 settings_limit,
51)
52from ..dependencies.threadpool import run_db_sync, run_model_discovery_sync
53from ..template_config import templates
54from .notes import _log_value_preview
56import asyncio
57import json
58import math
59import platform
60import threading
61import time
62from types import SimpleNamespace
63from typing import Any, Optional, Tuple, Annotated
64from datetime import UTC, datetime, timedelta, timezone
66import requests
68from loguru import logger
69from sqlalchemy.orm import Session
71from ...config.constants import DEFAULT_OLLAMA_URL
72from ...config.paths import (
73 get_custom_data_directory,
74 get_data_directory,
75 get_encrypted_database_path,
76)
77from ...constants import DEFAULT_SEARCH_TOOL
78from ...database.models.rate_limiting import RateLimitEstimate
79from ...database.models.settings import Setting, SettingType
80from ...database.session_context import get_user_db_session
81from ...database.encrypted_db import db_manager
82from ...database.sqlcipher_utils import retry_on_db_lock
83from ...llm.providers.base import normalize_provider
84from ...security.egress.policy import (
85 DEFAULT_EGRESS_SCOPE,
86 Decision,
87 EgressContext,
88 EgressScope,
89 PolicyDeniedError,
90 coerce_policy_bool,
91 context_from_snapshot,
92 effective_scope_for_display,
93 evaluate_llm_endpoint,
94 parse_user_egress_scope,
95 resolve_run_primary_engine,
96 unprotected_egress_allowed,
97)
98from ...security.egress.validators import (
99 first_egress_validation_error,
100)
101from ...utilities.db_utils import get_settings_manager
102from ...utilities.type_utils import to_positive_int, unwrap_setting
103from ...utilities.url_utils import normalize_url
105from ...settings.exceptions import SettingsValueError
106from ...settings.manager import (
107 UI_ELEMENT_TO_SETTING_TYPE,
108 _infer_ui_element,
109 check_env_setting,
110 get_typed_setting_value,
111 is_operator_only_setting,
112 is_valid_setting_key,
113 parse_boolean,
114)
115from ..services.settings_service import (
116 create_or_update_setting,
117 invalidate_settings_caches,
118 reschedule_document_jobs_if_needed,
119 reschedule_zotero_jobs_if_needed,
120 set_setting,
121)
123from ...security import safe_get
125# ``_is_empty_value`` is deliberately imported private (#6201): it is HALF
126# of the predicate ``redact_value`` gates masking on -- the other half is
127# ``is_sensitive_setting(key, ...)`` (``data_sanitizer.py``), which the
128# write-back merge below does NOT consult. The merge must restore a masked
129# leaf from at least the set of stored values that got masked in the first
130# place -- a merge rule NARROWER than the mask rule's emptiness half turns
131# an untouched GET-then-PUT round-trip into an unsatisfiable 400. Sharing
132# this one predicate instead of re-deriving it keeps that emptiness half in
133# lockstep; it does NOT make the merge equivalent to the mask rule as a
134# whole, since the merge restores on the submitted leaf being the sentinel
135# regardless of whether that leaf's own sub-key is sensitive -- e.g. a
136# stored ``{"port": 19530}`` is never masked (``port`` is not sensitive),
137# but a submitted ``{"port": "[REDACTED]"}`` over it is still restored to
138# ``19530``. Narrowing ``DEFAULT_SENSITIVE_KEYS`` / ``is_sensitive_setting``
139# therefore does NOT narrow what this merge will restore.
140from ...security.data_sanitizer import DataSanitizer, _is_empty_value
141from ..warning_checks import calculate_warnings
142from ..dependencies.json_body import json_body_error, read_json_dict
144# Create the router for settings
145router = APIRouter(prefix="/settings", tags=["settings"])
147# NOTE: Routes use username (not .get()) intentionally.
148# Depends(require_auth) guarantees the key exists; direct access fails
149# fast if the dependency is ever removed.
151# Settings with dynamically populated options (excluded from validation)
152DYNAMIC_SETTINGS = ["llm.provider", "llm.model", "search.tool"]
154# Settings whose changes trigger a warning recalculation.
155WARNING_AFFECTING_KEYS = frozenset(
156 [
157 "llm.provider",
158 "search.tool",
159 "search.iterations",
160 "search.questions_per_iteration",
161 "llm.local_context_window_size",
162 "llm.context_window_unrestricted",
163 "llm.context_window_size",
164 "policy.egress_scope",
165 "llm.require_local_endpoint",
166 "embeddings.require_local",
167 ]
168)
171def _shape_egress_scope_setting(key: str, metadata: Any) -> Any:
172 """Gate the operator-disabled "unprotected" escape hatch out of the
173 egress-scope setting's ``options`` and normalise its displayed
174 ``value`` wherever ``policy.egress_scope`` metadata is served to the
175 settings UI.
177 Ports the option-filtering / display-normalisation half of main's
178 ``web/routes/settings_routes.py::_shape_egress_scope_metadata``
179 (87537d9ec, "fix(security): operator-gate unprotected egress and
180 harden policy-sensitive consumers", #5148). Without this, the select's
181 ``options`` list (sourced from ``defaults/default_settings.json``,
182 which unconditionally lists "unprotected") is served to the browser
183 as-is, so the settings dashboard would offer the escape hatch even
184 when the operator never set ``LDR_POLICY_ALLOW_UNPROTECTED_EGRESS``.
186 NOTE: main's ``_shape_egress_scope_metadata`` also overlays an env-var
187 lock onto ``value``/``editable`` for the setting (a presentation cue
188 for *any* environment-overridden setting). That is a separate, much
189 broader feature — this branch's ``SettingsManager.get_all_settings()``
190 (see ``settings/manager.py``, ~line 977) already overlays
191 ``LDR_*`` env values onto ``value``/``editable`` uniformly for every
192 setting key, including ``policy.egress_scope`` — so it is intentionally
193 NOT duplicated here; doing so would just re-apply the same override a
194 second time.
195 """
196 if key != "policy.egress_scope" or not isinstance(metadata, dict):
197 return metadata
198 shaped = dict(metadata)
199 options = shaped.get("options")
200 if isinstance(options, list) and not unprotected_egress_allowed():
201 shaped["options"] = [
202 option
203 for option in options
204 if str(option.get("value") if isinstance(option, dict) else option)
205 .strip()
206 .lower()
207 != EgressScope.UNPROTECTED.value
208 ]
209 shaped["value"] = effective_scope_for_display(shaped.get("value"))
210 return shaped
213def _shape_pdf_storage_mode_setting(key: str, metadata: Any) -> Any:
214 """Hide the operator-gated unencrypted 'filesystem' PDF storage option.
216 Ports main's ``_shape_pdf_storage_mode_metadata``
217 (web/routes/settings_routes.py, fb49985aa, "operator-gate unprotected
218 egress and harden policy-sensitive consumers", #5148). Mirrors
219 ``_shape_egress_scope_setting``: when the operator gate
220 ``research_library.allow_filesystem_pdf_storage`` is off (the default),
221 the ``filesystem`` choice is stripped from the
222 ``research_library.pdf_storage_mode`` options served to the settings UI,
223 leaving the encrypted ``database`` default and ``none``.
224 Consumption-site coercion (``resolve_pdf_storage_mode``) is the actual
225 protection; this just keeps a disabled option out of the dropdown so a
226 user doesn't pick a value that is silently coerced away later.
227 """
228 if key != "research_library.pdf_storage_mode" or not isinstance(
229 metadata, dict
230 ):
231 return metadata
232 options = metadata.get("options")
233 if not isinstance(options, list): 233 ↛ 234line 233 didn't jump to line 234 because the condition on line 233 was never true
234 return metadata
235 # Lazy import to avoid pulling the research_library package in at
236 # settings-router module load time.
237 from ...research_library.services.pdf_storage_manager import (
238 filesystem_pdf_storage_allowed,
239 )
241 if filesystem_pdf_storage_allowed():
242 return metadata
243 shaped = dict(metadata)
244 shaped["options"] = [
245 option
246 for option in options
247 if str(option.get("value") if isinstance(option, dict) else option)
248 .strip()
249 .lower()
250 != "filesystem"
251 ]
252 return shaped
255def _apply_env_override(
256 settings_manager, key: str, value: Any, editable: bool
257) -> Tuple[Any, bool]:
258 """Overlay the LDR_* env-var value/editable state onto a single-key
259 lookup.
261 Reuses ``SettingsManager.get_all_settings()`` — the exact overlay
262 ``GET /settings/api`` and ``GET /settings/api/bulk`` already apply via
263 ``check_env_setting``/``get_typed_setting_value`` (see
264 ``settings/manager.py``, ~line 977) — instead of reimplementing that
265 logic here. Without this, ``GET /settings/api/{key}`` (single-key)
266 returned the stale DB value with ``editable=True`` for a setting an
267 operator had pinned via an LDR_* env var, while the bulk endpoints
268 correctly reported the effective value and ``editable=False`` for the
269 same key. This is a read/display fix only: writes to an env-locked
270 setting were already rejected server-side by
271 ``SettingsManager._is_environment_locked`` regardless of this bug.
272 """
273 effective = settings_manager.get_all_settings().get(key)
274 if effective is None:
275 return value, editable
276 return (
277 effective.get("value", value),
278 effective.get("editable", editable),
279 )
282def _filter_editable_settings(form_data: dict, db_session: Session) -> dict:
283 """Remove operator-locked and non-editable keys from *form_data* in place.
285 Operator-locked means an ``LDR_*`` environment variable pins the key
286 (``check_env_setting``). ``SettingsManager.set_setting`` refuses those
287 writes anyway, but it reports the refusal as a plain ``False``, which
288 the bulk write paths count as a *failed* save — so any ``LDR_*``
289 variable made every no-JS form POST flash "Saved with N setting(s)
290 failing" even though every editable key saved fine. Main dropped them
291 here instead; this restores that (#5978).
293 Returns a dict of *all* ``{key: Setting}`` records from the database
294 so callers can reuse it for further validation (e.g. egress-policy checks).
295 """
296 all_db_settings = {
297 setting.key: setting for setting in db_session.query(Setting).all()
298 }
300 non_editable_keys = [
301 key
302 for key in form_data.keys()
303 if is_operator_only_setting(key)
304 or check_env_setting(key) is not None
305 or (key in all_db_settings and not all_db_settings[key].editable)
306 ]
307 if non_editable_keys:
308 logger.bind(policy_audit=True).warning(
309 "Skipping operator-locked or non-editable settings: {}",
310 non_editable_keys,
311 )
312 for key in non_editable_keys:
313 del form_data[key]
315 return all_db_settings
318def _resolve_model_discovery_policy(
319 username: str,
320) -> tuple[EgressContext, dict[str, Any]]:
321 """Resolve the current egress policy before model cache or network access.
323 Ported from main's ``web/routes/settings_routes.py`` (87537d9ec,
324 "fix(security): operator-gate unprotected egress and harden
325 policy-sensitive consumers", #5148); the Flask module was deleted by
326 the FastAPI migration and this gate came across as a failing test only.
328 Fails CLOSED: any settings failure (or a non-dict snapshot) raises
329 ``PolicyDeniedError`` instead of falling back to "allow". The previous
330 ``_model_list_local_only`` helper returned False (allow) on error, so a
331 settings outage silently downgraded a local-only user to "list every
332 cloud provider" — which reads their stored API key and sends it to the
333 provider's model-listing endpoint.
334 """
335 settings_snapshot = None
336 try:
337 with get_user_db_session(username) as db_session:
338 if db_session: 338 ↛ 354line 338 didn't jump to line 354
339 settings_manager = get_settings_manager(db_session, username)
340 settings_snapshot = settings_manager.get_settings_snapshot(
341 strict=True
342 )
343 except PolicyDeniedError:
344 raise
345 except Exception:
346 logger.bind(policy_audit=True).warning(
347 "available-model policy settings unavailable"
348 )
349 raise PolicyDeniedError(
350 Decision(False, "settings_unavailable"),
351 target="available_models",
352 ) from None
354 if not isinstance(settings_snapshot, dict): 354 ↛ 355line 354 didn't jump to line 355 because the condition on line 354 was never true
355 raise PolicyDeniedError(
356 Decision(False, "settings_unavailable"),
357 target="available_models",
358 )
360 scope_raw = check_env_setting("policy.egress_scope")
361 if scope_raw is None:
362 scope_raw = settings_snapshot.get(
363 "policy.egress_scope", DEFAULT_EGRESS_SCOPE
364 )
365 if str(scope_raw).strip().lower() == EgressScope.BOTH.value: 365 ↛ 366line 365 didn't jump to line 366 because the condition on line 365 was never true
366 scope_raw = EgressScope.ADAPTIVE.value
367 parse_user_egress_scope(scope_raw)
368 primary = resolve_run_primary_engine(
369 settings_snapshot, default=DEFAULT_SEARCH_TOOL
370 )
371 return (
372 context_from_snapshot(
373 settings_snapshot,
374 primary,
375 username=username,
376 ),
377 settings_snapshot,
378 )
381def _model_discovery_provider_decision(
382 provider: str,
383 policy_context: EgressContext,
384 settings_snapshot: dict[str, Any],
385) -> Decision:
386 """Return the policy decision for a provider's configured endpoint."""
387 if not policy_context.require_local_llm:
388 return Decision(True, "no_local_requirement")
389 decision = evaluate_llm_endpoint(
390 normalize_provider(provider),
391 policy_context,
392 settings_snapshot=settings_snapshot,
393 )
394 if not decision.allowed:
395 logger.bind(policy_audit=True).info(
396 "available-model provider denied by egress policy",
397 provider=normalize_provider(provider),
398 reason=decision.reason,
399 )
400 return decision
403def _model_discovery_provider_allowed(
404 provider: str,
405 policy_context: EgressContext,
406 settings_snapshot: dict[str, Any],
407) -> bool:
408 """Return whether a provider's configured endpoint is allowed to list models."""
409 return _model_discovery_provider_decision(
410 provider, policy_context, settings_snapshot
411 ).allowed
414# Maps the policy's internal denial reason onto a short phrase describing
415# what the user would have to change. Anything unmapped falls back to the
416# control clause alone rather than leaking a raw reason token into the UI.
417_PROVIDER_DENIAL_CAUSES = {
418 "provider_cloud_only": "cloud-only provider",
419 "provider_url_unset": "no endpoint URL configured",
420 "url_malformed": "endpoint URL is malformed",
421 "provider_remote": "endpoint URL is not local",
422}
425def _configured_egress_scope(
426 settings_snapshot: dict[str, Any] | None,
427) -> EgressScope | None:
428 """Return the scope the user actually selected, or None if unknown.
430 ``EgressContext.scope`` is the RESOLVED scope: ``context_from_snapshot``
431 runs ``_resolve_adaptive_scope``, so the default ``adaptive`` selection
432 is stored as ``PRIVATE_ONLY`` whenever the primary search engine is
433 local. Naming that resolved value in the UI sends the user to a control
434 they never touched — their Egress Scope select still reads "Adaptive".
435 The configured value survives only on the settings snapshot, so read it
436 back from there, env-first — the same precedence
437 ``_resolve_model_discovery_policy`` and ``context_from_snapshot`` use
438 for this setting.
439 """
440 raw = check_env_setting("policy.egress_scope")
441 if raw is None:
442 if not isinstance(settings_snapshot, dict): 442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true
443 return None
444 raw = unwrap_setting(
445 settings_snapshot.get("policy.egress_scope", DEFAULT_EGRESS_SCOPE)
446 )
447 # Retired `both` is coerced to the protective default, as every other
448 # reader of this setting does (see context_from_snapshot).
449 if str(raw).strip().lower() == EgressScope.BOTH.value: 449 ↛ 450line 449 didn't jump to line 450 because the condition on line 449 was never true
450 raw = EgressScope.ADAPTIVE.value
451 try:
452 return parse_user_egress_scope(raw, disabled_unprotected="adaptive")
453 except PolicyDeniedError:
454 return None
457def _provider_disabled_reason(
458 decision: Decision,
459 policy_context: EgressContext,
460 settings_snapshot: dict[str, Any] | None,
461) -> str:
462 """Build the user-facing reason a provider is greyed out in the dropdown.
464 ``require_local_llm`` has three possible origins and the UI has to name
465 the right one, or the user goes hunting for a control that won't help:
466 an operator env lock, the ``private_only`` egress scope (which forces
467 the flag on at ``context_from_snapshot``), or the user's own
468 "Require Local LLM Endpoint" checkbox.
470 The scope arm has a fourth case hiding inside it: ``private_only`` may be
471 what ``adaptive`` RESOLVED to rather than what the user chose, so the
472 message is built from ``settings_snapshot`` — the same snapshot the
473 context was built from — and names the selection the user can actually
474 see. The parameter is required rather than defaulted: a caller with no
475 snapshot must say so with an explicit ``None`` (falling back to the
476 resolved scope) instead of silently getting the wrong control named.
477 """
478 env_value = check_env_setting("llm.require_local_endpoint")
479 # Truthiness must match what the policy enforces, not HTML-checkbox
480 # semantics: parse_boolean() calls "enabled" / "T" / "y" True, while
481 # context_from_snapshot() computes require_local_llm with the policy's
482 # own {"true","1","yes","on"} vocabulary. The looser parser reported a
483 # server lock that was not actually in force.
484 env_locked = env_value is not None and coerce_policy_bool(env_value)
486 notes = []
487 if not env_locked and policy_context.scope == EgressScope.PRIVATE_ONLY:
488 if _configured_egress_scope(settings_snapshot) == EgressScope.ADAPTIVE:
489 control = 'Blocked by egress scope "Adaptive"'
490 notes.append(
491 "resolved to Private only from your primary search engine"
492 )
493 else:
494 control = 'Blocked by egress scope "Private only"'
495 else:
496 control = 'Blocked by "Require Local LLM Endpoint"'
498 cause = _PROVIDER_DENIAL_CAUSES.get(decision.reason)
499 if cause:
500 notes.append(cause)
501 if env_locked:
502 notes.append("locked by the server")
503 return f"{control} ({'; '.join(notes)})" if notes else control
506def _provider_options_for_snapshot(options, policy_context, policy_snapshot):
507 """Describe provider availability without fetching models or credentials."""
508 provider_options = []
509 for option in options:
510 entry = dict(option)
511 decision = _model_discovery_provider_decision(
512 option["value"], policy_context, policy_snapshot
513 )
514 entry["disabled"] = not decision.allowed
515 entry["disabled_reason"] = (
516 _provider_disabled_reason(decision, policy_context, policy_snapshot)
517 if not decision.allowed
518 else None
519 )
520 provider_options.append(entry)
521 return provider_options
524def _get_setting_from_session(key: str | None, username: str, default=None):
525 """Helper to get a setting using the current session context.
527 A ``None`` key returns ``default``. ``SettingsManager.get_setting``
528 treats ``key=None`` as "return all settings"; this route helper fetches
529 a single named setting and must not inherit that bulk-read semantic.
530 Without the guard, callers iterating providers that declare
531 ``api_key_setting = None`` (LM Studio, Llama.cpp) would receive a dict
532 of every setting — leaking other providers' API keys.
533 """
534 if key is None:
535 return default
536 with get_user_db_session(username) as db_session:
537 if db_session:
538 settings_manager = get_settings_manager(db_session, username)
539 return settings_manager.get_setting(key, default)
540 return default
543def _model_discovery_settings_unchanged(db_session, fetched_settings):
544 """Compare discovery inputs with fresh rows in the cache write transaction."""
545 if not fetched_settings:
546 return True
547 rows = (
548 db_session.query(Setting)
549 .filter(Setting.key.in_(fetched_settings))
550 .populate_existing()
551 .all()
552 )
553 settings = {row.key: row for row in rows}
554 for key, fetched_value in fetched_settings.items():
555 row = settings.get(key)
556 current_value = get_typed_setting_value(
557 key,
558 row.value if row is not None else None,
559 str(row.ui_element) if row is not None else "text",
560 default="",
561 check_env=True,
562 )
563 if current_value != fetched_value:
564 return False
565 return True
568def validate_setting(
569 setting: Setting, value: Any
570) -> Tuple[bool, Optional[str]]:
571 """
572 Validate a setting value based on its type and constraints.
574 Args:
575 setting: The Setting object to validate against
576 value: The value to validate
578 Returns:
579 Tuple of (is_valid, error_message)
580 """
581 if setting.key == "search.fetch.summary_input_max_length":
582 try:
583 to_positive_int(value)
584 except ValueError:
585 return False, "Value must be a positive integer"
587 # Validate the exact summary integer before generic number conversion
588 # or the blank-input handling for other optional numeric settings.
589 raw_value = value
590 numeric_ui = setting.ui_element in ("number", "slider", "range")
591 # Blank form inputs explicitly clear optional numeric settings; they are
592 # normalized to None on write, not used as a fallback for bad numbers.
593 if numeric_ui and (
594 raw_value is None
595 or (isinstance(raw_value, str) and not raw_value.strip())
596 ):
597 return True, None
598 try:
599 value = get_typed_setting_value(
600 key=str(setting.key),
601 value=value,
602 ui_element=str(setting.ui_element),
603 default=None,
604 check_env=False,
605 )
606 except SettingsValueError as error:
607 if numeric_ui:
608 return False, "Value must be a number"
609 if setting.ui_element == "checkbox": 609 ↛ 610line 609 didn't jump to line 610 because the condition on line 609 was never true
610 return False, "Value must be a boolean"
611 return False, error.reason
613 # Validate based on UI element type
614 if setting.ui_element == "checkbox":
615 # After conversion, should be boolean
616 if not isinstance(value, bool):
617 return False, "Value must be a boolean"
619 elif setting.ui_element in ("number", "slider", "range"):
620 # After conversion, should be numeric
621 if (
622 isinstance(raw_value, bool)
623 or isinstance(value, bool)
624 or not isinstance(value, (int, float))
625 or (isinstance(value, float) and not math.isfinite(value))
626 ):
627 return False, "Value must be a number"
629 # Check min/max constraints if defined
630 if setting.min_value is not None and value < setting.min_value:
631 return False, f"Value must be at least {setting.min_value}"
632 if setting.max_value is not None and value > setting.max_value:
633 return False, f"Value must be at most {setting.max_value}"
635 elif setting.ui_element == "select":
636 # Check if value is in the allowed options
637 if setting.options:
638 # Skip options validation for dynamically populated dropdowns
639 if setting.key not in DYNAMIC_SETTINGS:
640 allowed_values = [
641 opt.get("value") if isinstance(opt, dict) else opt
642 for opt in list(setting.options) # type: ignore[arg-type]
643 ]
644 if value not in allowed_values:
645 return (
646 False,
647 f"Value must be one of: {', '.join(str(v) for v in allowed_values)}",
648 )
650 # All checks passed
651 return True, None
654def _strict_read_rejection(key: str, value: Any, ui_element: Any) -> str | None:
655 """Return why a strict settings read would reject this row, or None.
657 A write must not store a row that every effective read of the account
658 then refuses with ``SettingsValueError``. This applies the read's own
659 rules: the type must be supported and the value must convert to it.
660 Pass the value and the ui_element exactly as they will be stored. The
661 reason never contains the value.
662 """
663 try:
664 get_typed_setting_value(
665 key=key,
666 value=value,
667 ui_element=ui_element,
668 default=None,
669 check_env=False,
670 )
671 except SettingsValueError as error:
672 return error.reason
673 return None
676def coerce_setting_for_write(key: str, value: Any, ui_element: str) -> Any:
677 """Coerce an incoming value to the correct type before writing to the DB.
679 All web routes that save settings should use this function to ensure
680 consistent type conversion.
682 No JSON pre-parsing (``json.loads``) is needed here because:
683 - ``get_typed_setting_value`` already parses JSON strings internally
684 via ``_parse_json_value`` (for ``ui_element="json"``) and
685 ``_parse_multiselect`` (for ``ui_element="multiselect"``).
686 - For JSON API endpoints, ``await request.json()`` already delivers
687 dicts/lists as native Python objects.
688 - For ``ui_element="text"``, pre-parsing would corrupt data: a JSON
689 string like ``'{"k": "v"}'`` would become a dict, then ``str()``
690 would produce ``"{'k': 'v'}"`` (Python repr, not valid JSON).
691 """
692 if key == "search.fetch.summary_input_max_length":
693 try:
694 return to_positive_int(value)
695 except ValueError:
696 # Preserve invalid input for validate_setting to reject, rather
697 # than losing a fractional part or treating it as optional/null.
698 return value
700 # check_env=False: we are persisting a user-supplied value, not reading
701 # from an environment variable override. check_env=True (the default)
702 # would silently replace the user's value with an env var, which is
703 # incorrect on the write path.
704 numeric_ui = ui_element in ("number", "slider", "range")
705 if numeric_ui and isinstance(value, str) and not value.strip():
706 return None
707 if numeric_ui and isinstance(value, bool):
708 # bool is an int subclass, so float(True) would otherwise become 1 and
709 # erase the evidence validate_setting needs to reject type confusion.
710 return value
712 try:
713 coerced = get_typed_setting_value(
714 key=key,
715 value=value,
716 ui_element=ui_element,
717 default=None,
718 check_env=False,
719 )
720 except SettingsValueError:
721 # Preserve the submitted input so validate_setting can return the
722 # route's normal validation error. Never replace it with None/default,
723 # which would turn a malformed value into an accepted optional value.
724 return value
725 # Canonicalise the egress scope on the write path: the select options
726 # are exact lowercase strings, so a value arriving with surrounding
727 # whitespace or different casing (" STRICT ") would otherwise be
728 # rejected by validate_setting instead of being stored as "strict".
729 if key == "policy.egress_scope" and isinstance(coerced, str):
730 return coerced.strip().lower()
731 return coerced
734# Namespace validation for new setting creation via the web API (ported
735# from main's Flask settings_routes). Keys starting with any ALLOWED
736# prefix may be created; any prefix in BLOCKED takes precedence and is
737# rejected even if it also matches an allowed prefix. Existing keys
738# (updates) bypass this check — it only applies to creation of new DB
739# rows through the three write routes.
740ALLOWED_SETTING_PREFIXES = frozenset(
741 {
742 "app.",
743 "backup.",
744 "benchmark.",
745 "chat.",
746 "database.",
747 "document_scheduler.",
748 "embeddings.",
749 "focused_iteration.",
750 "general.",
751 "langgraph_agent.",
752 "llm.",
753 "local_search_",
754 "news.",
755 "notifications.",
756 "rag.",
757 "rate_limiting.",
758 "report.",
759 "research_library.",
760 "search.",
761 "ui.",
762 "web.",
763 "zotero.",
764 }
765)
766BLOCKED_SETTING_PREFIXES = frozenset(
767 {
768 "auth.",
769 "bootstrap.",
770 "db_config.",
771 "security.",
772 "server.",
773 "testing.",
774 }
775)
778# Recursion cap for EVERY container helper below -- the sentinel/depth
779# scanners, the merge, the positional-rebinding gate
780# (``_non_sentinel_fields_match``) and the type-aware equality it bottoms
781# out in (``_values_equal``). A helper in this family without the cap is
782# a 500 waiting to happen even when the others have it, because
783# ``_container_embeds_sentinel`` answers True unconditionally past its own
784# cap and therefore KEEPS the callers walking a too-deep value (#6201
785# round 7). json.loads accepts
786# JSON nested roughly 900 levels deep, but these helpers recurse once per
787# nesting level and hit Python's own call-stack RecursionError somewhere
788# between depth ~499 and ~600 (exact threshold depends on what else is on
789# the stack) -- well short of what a malicious/malformed payload can hand
790# them. A real setting's JSON value is never more than a handful of levels
791# deep, so a cap far below the crash point (and far above any legitimate
792# value) lets a too-deep payload be rejected cleanly by the callers below
793# instead of surfacing as an opaque 500 through the routes' broad
794# ``except Exception``.
795_MAX_CONTAINER_DEPTH = 64
798def _container_embeds_sentinel(
799 value: object, redaction_text: str, _depth: int = 0
800) -> bool:
801 """True when a dict/list contains the redaction sentinel as a substring
802 of ANY string leaf, exact-leaf-match or partially spliced into one.
804 Recursion counterpart of the ``str`` guards' ``redaction_text in value``
805 check, used for a container value under a setting that only matches the
806 broadened suffix arm (see ``_force_redact_strings``, which is what
807 produces such a container on the read side: every non-empty string leaf
808 becomes the bare sentinel, regardless of its own sub-key name). bool/int/
809 None leaves are never masked and so can never embed it.
811 Past ``_MAX_CONTAINER_DEPTH`` levels this conservatively returns True
812 (see ``_MAX_CONTAINER_DEPTH``'s comment) rather than recursing further:
813 a value nested that deep cannot be proven safe, and every caller here
814 treats "embeds the sentinel" as the signal to reject the write with a
815 400, which is the outcome we want for a pathological payload too.
816 """
817 if _depth > _MAX_CONTAINER_DEPTH:
818 return True
819 if isinstance(value, str):
820 return redaction_text in value
821 if isinstance(value, dict):
822 return any(
823 _container_embeds_sentinel(sub_val, redaction_text, _depth + 1)
824 for sub_val in value.values()
825 )
826 if isinstance(value, list):
827 return any(
828 _container_embeds_sentinel(item, redaction_text, _depth + 1)
829 for item in value
830 )
831 return False
834def _container_exceeds_max_depth(value: object, _depth: int = 0) -> bool:
835 """True when *value* nests a dict/list deeper than
836 ``_MAX_CONTAINER_DEPTH``, independent of whether it contains the
837 redaction sentinel anywhere.
839 Checked ahead of the sentinel guards so a too-deep payload with NO
840 sentinel in it gets its own error instead of
841 ``_redaction_sentinel_error``'s "contains the redaction placeholder"
842 message. ``_container_embeds_sentinel`` and ``_merge_redacted_container``
843 both conservatively treat anything past the cap as "embeds it" /
844 "unresolvable" regardless of actual content -- the right call for
845 deciding to reject a payload that can't be proven safe, but the wrong
846 reason to report when nothing in it actually matched the sentinel.
847 """
848 if _depth > _MAX_CONTAINER_DEPTH:
849 return True
850 if isinstance(value, dict):
851 return any(
852 _container_exceeds_max_depth(sub_val, _depth + 1)
853 for sub_val in value.values()
854 )
855 if isinstance(value, list):
856 return any(
857 _container_exceeds_max_depth(item, _depth + 1) for item in value
858 )
859 return False
862def _container_too_deep_error() -> str:
863 """Explain the 400 raised when a dict/list value nests deeper than
864 ``_MAX_CONTAINER_DEPTH`` -- a distinct condition from embedding the
865 redaction sentinel and reported with its own message (see
866 ``_container_exceeds_max_depth``), instead of the misleading
867 "contains the redaction placeholder" wording a too-deep payload with
868 no sentinel in it used to get by reusing ``_redaction_sentinel_error``.
869 """
870 return (
871 "Value is nested too deeply to validate "
872 f"(max {_MAX_CONTAINER_DEPTH} levels) — flatten it before saving."
873 )
876def _container_rejection_error(ui_element: str | None, value: object) -> str:
877 """Pick the right 400 message for a rejected dict/list write: the
878 depth-cap message when *value* nests past ``_MAX_CONTAINER_DEPTH``
879 (the cap itself is why it was rejected, whether or not it happens to
880 also contain the sentinel), otherwise the usual
881 redaction-sentinel message.
882 """
883 if isinstance(value, (dict, list)) and _container_exceeds_max_depth(value):
884 return _container_too_deep_error()
885 return _redaction_sentinel_error(ui_element)
888def _values_equal(a: object, b: object, _depth: int = 0) -> bool:
889 """Type-aware equality for comparing a submitted/merged value against
890 its stored counterpart.
892 Plain ``==`` treats ``1 == True`` as equal (``bool`` is an ``int``
893 subclass), so a checkbox submitted as ``1`` over a stored ``True`` --
894 or ``{"enabled": 1}`` over a stored ``{"enabled": True}`` -- would
895 look byte-for-byte identical to
896 ``_resolve_container_secret_write``'s no-op check and silently
897 discard a genuine type change instead of persisting it (#6201).
898 ``bool`` is therefore checked FIRST, before the numeric case, because
899 it IS an ``int``; dict/list are walked structurally rather than
900 compared with their own ``==`` so a type mismatch nested inside a
901 container is caught too, not just at the top level.
903 ``int`` vs ``float`` is deliberately NOT a difference. This predicate
904 also decides ``_non_sentinel_fields_match``, the gate that must see
905 every visible field of a list item unchanged before a masked secret
906 is restored by position -- and requiring ``type(a) is type(b)`` there
907 made an ordinary browser round-trip UNSATISFIABLE. A stored
908 ``[{"url": "a", "api_key": ..., "timeout": 30.0}, {...}]`` is served
909 to the client with both keys masked; JavaScript has no ``30.0`` (its
910 only number type is a double that serializes as ``30``), and
911 ``settings.js`` additionally ``parseInt``s a decimal-free numeric
912 before submitting. The client therefore CANNOT echo back a value
913 ``type(a) is type(b)`` accepts, so every edit to a >=2-item list
914 carrying a masked secret 400'd with "contains the redaction
915 placeholder" no matter what the user did (#6201 round 5). Comparing
916 numerics by value costs nothing the guard actually relies on: an
917 attacker cannot use "same number, other numeric type" to disguise a
918 reorder, because two list entries that differ ONLY in whether a
919 number is stored as ``30`` or ``30.0`` are indistinguishable to any
920 JSON client to begin with.
922 Depth-capped like the other container walkers (see
923 ``_MAX_CONTAINER_DEPTH``): past the cap this answers "not equal"
924 rather than recursing into a stack overflow. That is the fail-closed
925 answer for both callers -- ``_non_sentinel_fields_match`` treats it as
926 "cannot prove this position is unchanged" (so a positional secret
927 restore is refused) and ``_resolve_container_secret_write`` treats it
928 as "not a no-op" (so nothing is silently swallowed). Without the cap a
929 ~7 KB payload nested ~1000 levels deep against a stored value of the
930 same shape raised an uncaught ``RecursionError`` that the write routes'
931 broad ``except Exception`` turned into a 500 (#6201 round 7).
932 """
933 if _depth > _MAX_CONTAINER_DEPTH:
934 return False
935 if isinstance(a, dict) or isinstance(b, dict):
936 if not (isinstance(a, dict) and isinstance(b, dict)): 936 ↛ 937line 936 didn't jump to line 937 because the condition on line 936 was never true
937 return False
938 if a.keys() != b.keys():
939 return False
940 return all(_values_equal(a[k], b[k], _depth + 1) for k in a)
941 if isinstance(a, list) or isinstance(b, list):
942 if not (isinstance(a, list) and isinstance(b, list)): 942 ↛ 943line 942 didn't jump to line 943 because the condition on line 942 was never true
943 return False
944 if len(a) != len(b): 944 ↛ 945line 944 didn't jump to line 945 because the condition on line 944 was never true
945 return False
946 return all(_values_equal(x, y, _depth + 1) for x, y in zip(a, b))
947 # bool BEFORE the numeric case: `bool` is an `int`, so a plain `==`
948 # here would call `1` unchanged from a stored `True`.
949 if isinstance(a, bool) or isinstance(b, bool):
950 return type(a) is type(b) and a == b
951 # Neither side is a bool by now, so `int`/`float` compare by VALUE:
952 # `30 == 30.0` is the same number, and a JSON client has no way to
953 # tell the two apart (see the docstring).
954 return a == b
957def _non_sentinel_fields_match(
958 value: object,
959 stored_value: object,
960 redaction_text: str,
961 _depth: int = 0,
962) -> bool:
963 """True when every leaf of *value* that is NOT the redaction sentinel
964 equals ``stored_value``'s leaf at the same path -- i.e. nothing
965 visible changed except the masked leaf(s).
967 Used ONLY to gate positional secret restoration inside a LIST (see the
968 list branch of ``_merge_redacted_container``): a same-length
969 permutation of a list of dicts is indistinguishable from an in-place
970 edit by position alone -- ``zip()`` pairs each submitted item with
971 whatever stored item shares its index, not with the item it actually
972 came from. Given stored ``[{url:a,api_key:K1},{url:b,api_key:K2}]``, a
973 client that reorders the two entries and resubmits both with their
974 (masked) ``api_key`` untouched would otherwise have K1 silently
975 rebound to ``b`` and K2 to ``a`` -- restoring the WRONG secret under
976 each url, not the one it actually belonged to (#6201). Requiring every
977 visible field at that index to match before trusting the position
978 catches this: a genuine in-place edit leaves everything but the
979 secret leaf(s) unchanged, so it still matches; a reorder does not.
981 This deliberately does NOT extend the "edit a sibling next to a
982 restored secret" allowance ``_merge_redacted_container`` documents for
983 a single dict/object -- that allowance is safe there because there is
984 only one candidate object to restore into. Inside a list, the same
985 allowance would make a sibling edit indistinguishable from a reorder,
986 which is exactly the hole this closes. The accepted consequence: an
987 ordinary edit to a NON-secret sibling field of a list item that also
988 carries an untouched masked secret at the same index is rejected
989 along with a genuine reorder -- the caller must split that into two
990 writes (clear/replace the secret first, or edit the sibling via a
991 request that doesn't also carry the sentinel) rather than combining
992 them in one PUT.
994 Depth-capped like every other container walker here (see
995 ``_MAX_CONTAINER_DEPTH``). Past the cap this answers False -- "nothing
996 proves this position still holds the same entry" -- which makes the
997 caller reject the item rather than positionally rebind a secret into
998 a container it could not finish inspecting. This helper and
999 ``_values_equal`` were the last two in the merge path without a cap,
1000 and ``_merge_redacted_container``'s list branch called this one with
1001 no depth at all: a ~7 KB list whose first item nested ~1000 levels
1002 deep against a stored list of the same shape raised an uncaught
1003 ``RecursionError`` and surfaced as a 500 through the write routes'
1004 broad ``except Exception`` -- the exact outcome ``_MAX_CONTAINER_DEPTH``
1005 exists to prevent (#6201 round 7). Note the depth cap alone does not
1006 end the walk: ``_container_embeds_sentinel`` answers True
1007 unconditionally past its own cap, so this gate is still consulted for
1008 a too-deep item and must terminate on its own.
1009 """
1010 if _depth > _MAX_CONTAINER_DEPTH:
1011 return False
1012 if isinstance(value, str) and value == redaction_text:
1013 return True
1014 if isinstance(value, dict):
1015 if not isinstance(stored_value, dict): 1015 ↛ 1016line 1015 didn't jump to line 1016 because the condition on line 1015 was never true
1016 return False
1017 # The submitted item's visible keys must exactly match the stored
1018 # item's keys, not just be a subset of them. A subset check lets a
1019 # client RENAME or DROP a visible field (e.g. resubmit ``url`` as
1020 # ``endpoint``, or omit it) to make an otherwise-mismatched item
1021 # look like a match: the renamed/added key isn't in ``stored_value``
1022 # so a subset walk would skip it, and the dropped key is simply
1023 # never visited from ``value.items()``. Either way position gets
1024 # trusted with nothing left to disprove it -- the exact bypass this
1025 # function exists to close (#6201 round 3 follow-up). Requiring
1026 # equal key sets also matches this module's own rule that adding a
1027 # field to a list item alongside a masked secret must be split into
1028 # two writes, rather than silently letting the extra field ride
1029 # along as if it always existed at this position.
1030 if value.keys() != stored_value.keys():
1031 return False
1032 for sub_key, sub_val in value.items():
1033 if not _non_sentinel_fields_match(
1034 sub_val, stored_value[sub_key], redaction_text, _depth + 1
1035 ):
1036 return False
1037 return True
1038 if isinstance(value, list): 1038 ↛ 1039line 1038 didn't jump to line 1039 because the condition on line 1038 was never true
1039 if not isinstance(stored_value, list) or len(stored_value) != len(
1040 value
1041 ):
1042 return False
1043 return all(
1044 _non_sentinel_fields_match(
1045 sub_val, sub_stored, redaction_text, _depth + 1
1046 )
1047 for sub_val, sub_stored in zip(value, stored_value)
1048 )
1049 return _values_equal(value, stored_value, _depth)
1052def _merge_redacted_container(
1053 value: object, stored_value: object, redaction_text: str, _depth: int = 0
1054) -> Tuple[object, bool]:
1055 """Merge a dict/list write against its stored counterpart, restoring
1056 the real secret under every leaf that is EXACTLY the redaction
1057 sentinel while keeping every other leaf's submitted value as-is -- so
1058 editing a non-secret sibling (or leaving a DIFFERENT secret leaf
1059 untouched) is saved instead of the whole container being rejected.
1061 ``redact_value`` (``security/data_sanitizer.py``) masks a container by
1062 SUB-KEY: in a MIXED container -- e.g. ``{"api_key": "sk-...",
1063 "description": "My engine", "requires_api_key": true}`` -- only the
1064 sensitive-named leaf becomes the sentinel; ordinary siblings stay
1065 plain. A caller that GETs, edits ``description``, and PUTs the result
1066 back therefore submits a container where exactly one leaf is the
1067 sentinel and the rest are ordinary values -- not the "every maskable
1068 leaf is the sentinel" shape round 5's fix required of the WHOLE
1069 container before treating it as anything but a corrupted edit.
1070 Rejecting that outright turns a legitimate edit into an unsatisfiable
1071 400: the client can never construct a request this endpoint will
1072 accept, because it cannot see the real secret to resubmit it and any
1073 masked-leaf resubmission was treated as "embeds the sentinel". This
1074 function resolves it leaf-by-leaf instead of all-or-nothing.
1076 ACCEPTED RISK, DELIBERATE (post-#6201 review, item a): this leaf-by-
1077 leaf merge means a caller who has never seen the real secret can still
1078 edit a NON-secret sibling of it in the SAME write -- e.g. change
1079 ``search.engine.web.<engine>.url`` while ``api_key`` stays
1080 ``[REDACTED]`` -- and the merge restores the real key under the NEW
1081 url. A round of this feature's own review (round 5) rejected mixed
1082 containers outright specifically to prevent this; this round reverses
1083 that call because the outright rejection made the endpoint
1084 unsatisfiable for the overwhelmingly common case (any GET-then-edit of
1085 a mixed container at all, secret untouched). This is a real,
1086 consciously-accepted trade rather than an oversight: egress/SSRF
1087 validation for registered settings (``security/egress/validators.py``)
1088 keys off FIXED, known setting names, not the contents of arbitrary
1089 user-defined JSON, so this does not by itself hand a caller a new way
1090 to bypass egress policy for a setting the validators inspect. It DOES
1091 mean a masked secret can end up silently repointed at a different
1092 destination than the one it was originally saved under, without the
1093 caller ever needing to know the secret's value. If that trade
1094 ever needs tightening, the narrowing applied to LIST items below (see
1095 the list branch's docstring section) -- requiring every visible field
1096 at a position to match before trusting a positional secret restore --
1097 is the template: the same "verify nothing else changed" check could be
1098 applied to a bare dict's siblings too, at the cost of reintroducing
1099 the unsatisfiable-400 problem this round fixed.
1101 Returns ``(merged_value, rejected)``. ``rejected`` is True when a leaf
1102 cannot be resolved safely:
1104 * the leaf equals the sentinel exactly, but ``stored_value`` is
1105 itself empty (absent, ``""``, ``" "``, ``None``, ``[]`` or ``{}``)
1106 at that path -- there is nothing to "keep", so persisting it would
1107 write the literal placeholder text as the value. This checks only
1108 ``stored_value``'s emptiness, the same half of ``redact_value``'s
1109 gate that ``_is_empty_value`` implements -- it does NOT also check
1110 that the leaf's own sub-key is sensitive, so the restore set is a
1111 SUPERSET of the mask set: a stored ``{"port": 19530}`` is never
1112 masked on read (``port`` is not sensitive), but a submitted
1113 ``{"port": "[REDACTED]"}`` over it still restores ``19530`` here.
1114 Narrowing ``DEFAULT_SENSITIVE_KEYS`` / ``is_sensitive_setting``
1115 therefore does not narrow what this merge will restore;
1116 * the leaf merely CONTAINS the sentinel as a substring without being
1117 an exact match (e.g. a stale client's ``"[REDACTED],new-value"``)
1118 -- an edit, but a corrupted one that would splice the placeholder
1119 text into the stored value, the same corrupted-edit case the
1120 scalar arm of ``_embeds_redaction_sentinel`` rejects.
1122 A list is walked positionally only when the submitted and stored lists
1123 are the same length; a length change means there is no reliable
1124 correspondence between items, so any sentinel occurring in it can't be
1125 resolved and the whole list is rejected the same way an orphaned leaf
1126 is (matching the "can't confirm, so treat as unresolvable" stance the
1127 other container helpers take). A dict key absent from ``stored_value``
1128 is not itself an error -- it is an ordinary new field unless its own
1129 value happens to be the sentinel, which the orphan case above catches.
1131 Equal length is NOT sufficient on its own to trust a list's positions,
1132 though: a same-length PERMUTATION of a list of dicts is invisible to a
1133 plain ``zip()`` walk. Given stored ``[{url:a,api_key:K1},
1134 {url:b,api_key:K2}]``, a client that swaps the two entries and
1135 resubmits both with their ``api_key`` still masked would otherwise get
1136 K1 silently restored under ``b`` and K2 under ``a`` -- the wrong
1137 secret rebound to the wrong entry, not a rejection (#6201). Before
1138 restoring a sentinel-bearing dict/list item positionally, this walks
1139 it through ``_non_sentinel_fields_match`` first: every field at that
1140 index NOT equal to the sentinel must match the stored item at the same
1141 index, proving nothing but the masked leaf(s) changed there. A
1142 mismatch means the position can't be trusted to identify the same
1143 real-world entry the secret was read from, so that item's sentinel
1144 leaves are treated as unresolvable (rejected) rather than restored
1145 into what may be a different entry. This intentionally narrows the
1146 "edit a sibling next to a restored secret" allowance documented above
1147 to the single-object case -- inside a list it is indistinguishable
1148 from a reorder, so it isn't extended there.
1150 EXCEPTION: a single-element list (``len(value) == 1``) skips this
1151 sibling-match gate entirely. The gate exists solely to rule out a
1152 same-length reorder, and a list of one has no other position for its
1153 one item to have come from or gone to -- there is no reorder for a
1154 sibling edit to be confused with, so the "edit a sibling next to a
1155 restored secret" allowance the single-object dict case gets above
1156 applies here too.
1157 """
1158 # NOTE (fragility, deliberate): past the cap this does NOT reject on its
1159 # own -- it returns whatever ``_container_embeds_sentinel`` says, and that
1160 # helper's own cap makes it answer True unconditionally past this depth.
1161 # So the clean 400 a too-deep payload gets today is produced by that
1162 # helper's fail-closed default, not by this line. The composition is
1163 # correct (verified at 65/900/3000 levels), but it means a future edit
1164 # that makes ``_container_embeds_sentinel`` return a real answer past the
1165 # cap -- or drops its cap because "the callers already cap" -- would
1166 # silently turn a too-deep payload into an ACCEPTED merge here rather
1167 # than a rejection, with no test naming this line. Anything that touches
1168 # either cap must re-check this pair together; ``return value, True`` is
1169 # the equivalent that does not depend on the other helper, kept as a
1170 # call rather than a literal only so the two stay visibly coupled.
1171 if _depth > _MAX_CONTAINER_DEPTH:
1172 return value, _container_embeds_sentinel(value, redaction_text, _depth)
1174 if isinstance(value, dict):
1175 stored_dict = stored_value if isinstance(stored_value, dict) else {}
1176 merged: dict = {}
1177 rejected = False
1178 for sub_key, sub_val in value.items():
1179 merged_val, sub_rejected = _merge_redacted_container(
1180 sub_val, stored_dict.get(sub_key), redaction_text, _depth + 1
1181 )
1182 merged[sub_key] = merged_val
1183 rejected = rejected or sub_rejected
1184 return merged, rejected
1186 if isinstance(value, list):
1187 if isinstance(stored_value, list) and len(stored_value) == len(value):
1188 merged_list = []
1189 rejected = False
1190 for sub_val, sub_stored in zip(value, stored_value):
1191 # A single-element list has exactly one possible position
1192 # for its one item -- there is no OTHER index a reorder
1193 # could have moved it from or to, so the "position can't
1194 # be trusted" rationale below (which exists to catch a
1195 # same-length reorder) cannot apply. Skip the sibling-
1196 # match gate entirely for len == 1 rather than rejecting
1197 # an ordinary sibling edit next to a masked secret that
1198 # has no reorder to be confused with.
1199 if (
1200 len(value) != 1
1201 and _container_embeds_sentinel(
1202 sub_val, redaction_text, _depth + 1
1203 )
1204 and not _non_sentinel_fields_match(
1205 sub_val, sub_stored, redaction_text, _depth + 1
1206 )
1207 ):
1208 # A visible (non-sentinel) field at this index differs
1209 # from the stored item at the same index -- position
1210 # cannot be trusted to identify the same entry the
1211 # secret leaf was read from (e.g. a same-length
1212 # reorder). Reject rather than positionally rebind the
1213 # secret to a possibly different entry (#6201).
1214 merged_list.append(sub_val)
1215 rejected = True
1216 continue
1217 merged_val, sub_rejected = _merge_redacted_container(
1218 sub_val, sub_stored, redaction_text, _depth + 1
1219 )
1220 merged_list.append(merged_val)
1221 rejected = rejected or sub_rejected
1222 return merged_list, rejected
1223 return value, _container_embeds_sentinel(value, redaction_text, _depth)
1225 if isinstance(value, str):
1226 if value == redaction_text:
1227 # Restore from ANY non-empty stored value, not just a string
1228 # (#6201/#6213 item c). ``redact_value`` masks a nested leaf to
1229 # the bare sentinel for any non-empty value whose sub-key is an
1230 # exact ``DEFAULT_SENSITIVE_KEYS`` match -- a list, dict, int or
1231 # bool included -- so requiring ``isinstance(stored_value, str)``
1232 # here made the write-back rule STRICTER than the masking rule:
1233 # for all 18 exact sensitive leaf names, a stored list/dict/int/
1234 # bool leaf was masked on GET and then refused on PUT, an
1235 # unsatisfiable 400 the client could never construct a passing
1236 # request for (it cannot see the value the GET collapsed to one
1237 # string), and on ``save_all_settings`` a rejection rolls back
1238 # the WHOLE batch. Mirroring ``_is_empty_value`` -- the exact
1239 # gate ``redact_value`` masks on -- makes the round-trip a no-op
1240 # again. Every other rejection is unchanged: an empty stored
1241 # value ("", " ", None, [], {}) was never masked, so a sentinel
1242 # over it is still an orphan (below); a missing sub-key arrives
1243 # here as ``None`` from ``stored_dict.get(...)`` and is still
1244 # rejected; and a sentinel EMBEDDED in a longer string is still
1245 # the corrupted-edit case.
1246 if not _is_empty_value(stored_value):
1247 return stored_value, False
1248 return value, True
1249 if redaction_text in value:
1250 return value, True
1251 return value, False
1253 return value, False
1256def _resolve_container_secret_write(
1257 value: object, stored_value: object, redaction_text: str
1258) -> Tuple[object, bool, bool]:
1259 """Decide the outcome of a dict/list write that may carry the
1260 redaction sentinel.
1262 Returns ``(effective_value, is_noop, is_rejected)``:
1264 * No sentinel anywhere in *value* -- pass it through untouched
1265 (``is_noop=False``, ``is_rejected=False``); there is nothing for
1266 this guard to do with an ordinary container write.
1267 * The sentinel is present but there is no ``stored_value`` to
1268 restore a masked leaf from -- rejected, same as before this only
1269 merges against a row that actually exists.
1270 * The sentinel is present and every occurrence resolves against
1271 ``stored_value`` (see ``_merge_redacted_container``) -- the merged
1272 container is the value to persist. If it is byte-for-byte
1273 identical to what's already stored, the write is a no-op (covers
1274 both the fully-masked round-trip and a partially-masked one where
1275 nothing actually changed); otherwise it is a genuine edit and the
1276 caller must persist the MERGED value, not the raw submitted
1277 *value*, which still carries the sentinel on its untouched leaves.
1278 * Any leaf cannot be resolved (see ``_merge_redacted_container``) --
1279 rejected.
1280 """
1281 if not _container_embeds_sentinel(value, redaction_text):
1282 return value, False, False
1283 if stored_value is None:
1284 return value, False, True
1285 merged, rejected = _merge_redacted_container(
1286 value, stored_value, redaction_text
1287 )
1288 if rejected:
1289 return value, False, True
1290 # Type-aware equality (#6201): plain ``==`` would call
1291 # ``{"enabled": 1}`` unchanged from a stored ``{"enabled": True}`` and
1292 # silently discard the type change as a false no-op.
1293 if _values_equal(merged, stored_value):
1294 return merged, True, False
1295 return merged, False, False
1298def _coerced_for_guard(
1299 key: str, ui_element: str | None, value: object
1300) -> object:
1301 """Return *value* as the sentinel guards should see it, coerced to its
1302 stored shape when a JSON-STRING encoding would otherwise hide a masked
1303 container from them.
1305 ``PUT /settings/api/<key>`` and both bulk-save routes accept a "json"
1306 ui_element's value either as a native dict/list -- caught by the
1307 container arm of ``_is_secret_empty_noop`` / ``_embeds_redaction_sentinel``
1308 -- or as its JSON-STRING encoding, which ``coerce_setting_for_write`` ->
1309 ``get_typed_setting_value`` -> ``_parse_json_value``
1310 (``settings/manager.py``) decodes into the same dict, but only AFTER
1311 every guard call site runs its checks. A masked container submitted in
1312 string form therefore sails past the container arm (which only fires
1313 on an actual ``dict``/``list``) and falls through to the string arm,
1314 which is gated on the OUTER key's sensitivity -- exactly the
1315 "non-sensitive outer key with a sensitive nested leaf" case the
1316 container arm exists to close. Coercing here, before the guards run,
1317 closes that gap the same way the create path's registered-default
1318 branch already does (it coerces before its own guard call).
1320 Only a ``str`` is re-coerced, and the coerced result is used only if
1321 it actually becomes a dict/list: an already-dict/list *value* is
1322 already handled correctly by the container arm as-is, and running it
1323 back through a non-container ``ui_element`` (e.g. "text": ``str(value)``)
1324 would destroy that shape instead of revealing one. This return value
1325 is used ONLY to decide what the guards see and, when a sentinel is
1326 found, to compute the merged replacement value -- callers must keep
1327 persisting their own *value* (or that merge result) unchanged in every
1328 other case, never this function's return value directly, so an
1329 ordinary JSON-string write (no sentinel involved) is unaffected.
1331 A row's ``ui_element`` can itself disagree with its actual stored
1332 shape -- e.g. a ``PUT``-created key with no ``ui_element`` defaults to
1333 "text" (``models/settings.py``) even when a caller stores a dict/list
1334 value under it. For "text"/"textarea"/"password"/"select", the
1335 ui_element-based coercion above NEVER becomes a dict/list (their
1336 target type IS ``str``), so a masked container submitted in
1337 JSON-STRING form under one of THOSE elements would still sail past
1338 the container arm even after the coercion above (#6201). When the
1339 ui_element-based attempt didn't produce a container, this also tries
1340 a plain JSON decode of the same string independent of ``ui_element``
1341 entirely -- the row's ACTUAL stored shape decides whether this is a
1342 container write in disguise, not its declared type. Quiet on failure
1343 (unlike ``settings/manager.py``'s ``_parse_json_value``, which logs a
1344 warning on every non-JSON string): an ordinary "text" setting holding
1345 an ordinary non-JSON string is the overwhelmingly common case here and
1346 must not spam the log.
1348 The BARE sentinel (*value* exactly equal to the redaction text) is
1349 special-cased to skip coercion entirely and pass through unchanged.
1350 ``coerce_setting_for_write("multiselect", "[REDACTED]", ...)`` ->
1351 ``_parse_multiselect`` (``settings/manager.py``) sees a string starting
1352 with "[" that fails ``json.loads`` (the sentinel isn't valid JSON) and
1353 falls back to its comma-separated split, which -- since there is no
1354 comma -- wraps the whole string into a ONE-ELEMENT LIST,
1355 ``["[REDACTED]"]``. That list then looks like a malformed/rejected
1356 container write (a length-mismatch against a stored list of any other
1357 size) instead of the ordinary round-tripped sentinel it actually is,
1358 turning what used to be a 200 no-op into a 400 for a sensitive
1359 ``multiselect`` setting (#6201 item 6). The exact-match scalar arm of
1360 ``_is_secret_empty_noop`` / ``_embeds_redaction_sentinel`` handles a
1361 bare-string sentinel for a SENSITIVE key under every ui_element (both
1362 scalar arms are ``is_sensitive_setting``-gated -- a bare sentinel
1363 submitted for a NON-sensitive container row still persists verbatim
1364 over the stored container, which those arms do not cover), so there is
1365 nothing this coercion needs to reveal for that case.
1367 A SPLICED sentinel on a ``multiselect`` -- e.g. ``"[REDACTED],new"``
1368 from a stale client that rendered the mask and had a value appended --
1369 must skip the coercion for the same reason, and this one is a
1370 data-loss bug rather than a usability one. ``_parse_multiselect``'s
1371 comma-split fallback fires again (the whole string still isn't valid
1372 JSON) and turns the SPLICED scalar into ``["[REDACTED]", "new"]``,
1373 whose first item is now an EXACT sentinel match. The container arm
1374 then merges it positionally against a stored list of the same length
1375 and persists ``["A", "new"]`` with a 200 -- silently DESTROYING the
1376 stored ``"B"`` -- so the "contains the sentinel but is not equal to it
1377 => hard 400" rule that ``_merge_redacted_container`` documents (and
1378 that the scalar arm enforces, as the base branch did here) never fires
1379 at all. Splitting a spliced scalar into a list manufactures the exact
1380 match out of a corrupted edit, so malformed JSON carrying the sentinel
1381 is handed to the guards AS A STRING for this ui_element; the scalar arm
1382 of ``_embeds_redaction_sentinel`` then rejects it with the usual 400
1383 (#6201 round 7). Only ``multiselect`` needs this: ``json`` decodes
1384 with ``json.loads`` alone, which never invents list structure out of a
1385 non-JSON string. Valid JSON containers under ``multiselect`` must still
1386 be decoded before the guards run, so nested masks restore the stored
1387 secrets instead of being persisted literally.
1388 """
1389 if not isinstance(value, str) or not ui_element:
1390 return value
1391 if value == DataSanitizer.REDACTION_TEXT:
1392 return value
1393 # ``multiselect`` is the one ui_element whose coercion SPLITS a string
1394 # on commas, so a spliced sentinel ("[REDACTED],new") comes back as a
1395 # list whose first item is an exact sentinel match -- laundering a
1396 # corrupted edit into a merge that silently drops the stored items the
1397 # client did not resend. Keep it a string so the scalar arm can reject
1398 # it (see the docstring).
1399 if ui_element == "multiselect" and DataSanitizer.REDACTION_TEXT in value:
1400 # Strict JSON reveals real containers without the comma-split
1401 # fallback manufacturing a container from a corrupted scalar.
1402 try:
1403 parsed = json.loads(value)
1404 except (json.JSONDecodeError, ValueError, RecursionError):
1405 return value
1406 return parsed if isinstance(parsed, (dict, list)) else value
1407 # Only "json" and "multiselect" can actually decode a string into a
1408 # dict/list via ``coerce_setting_for_write`` -- every other
1409 # ui_element's converter (``float``, ``bool``, ``str``, ...) can never
1410 # produce one. Calling it for those elements anyway would run
1411 # ``coerce_setting_for_write`` a SECOND time on the exact same
1412 # (key, value, ui_element) once the caller coerces again for the
1413 # actual persisted write below (every string write on an existing
1414 # setting would otherwise pay for this twice): a failing conversion
1415 # (e.g. "abc" on a "number" row) would log its warning twice, and for
1416 # a "number"/"range" row this ran ``float(x)`` on the RAW submission
1417 # BEFORE the sentinel guards below get a chance to run. That used to
1418 # log the raw typed submission at WARNING on conversion failure --
1419 # exactly what the guards below exist to prevent from happening
1420 # unchecked. ``get_typed_setting_value`` no longer logs a setting
1421 # VALUE at all (#6201 round 6), so the disclosure half of this
1422 # rationale is now closed at the converter; the duplicate-work and
1423 # duplicate-warning halves stand on their own.
1424 # Skipping the call here for non-container-capable elements removes
1425 # that channel entirely; those elements still fall through to the
1426 # ui_element-independent JSON-shape check below, which is what
1427 # actually matters for detecting a container submitted under a
1428 # mismatched ui_element (see the docstring).
1429 if ui_element in ("json", "multiselect"):
1430 coerced = coerce_setting_for_write(
1431 key=key, value=value, ui_element=ui_element
1432 )
1433 if isinstance(coerced, (dict, list)):
1434 return coerced
1435 stripped = value.strip()
1436 if stripped and stripped[0] in "{[":
1437 try:
1438 parsed = json.loads(stripped)
1439 except (json.JSONDecodeError, ValueError, RecursionError):
1440 return value
1441 if isinstance(parsed, (dict, list)): 1441 ↛ 1443line 1441 didn't jump to line 1443 because the condition on line 1441 was always true
1442 return parsed
1443 return value
1446def _reconciled_container_value_for_write(
1447 raw_value: object, guard_value: object, stored_value: object
1448) -> Tuple[object, bool]:
1449 """Return the value to actually PERSIST for a write that has already
1450 passed both sentinel guards (``_is_secret_empty_noop`` returned False
1451 and ``_embeds_redaction_sentinel`` returned False for *guard_value*).
1453 Returns ``(value_to_persist, is_reconciled_container)``.
1455 For a scalar, or a dict/list that embeds no sentinel at all, this is
1456 ``(raw_value, False)`` -- including when *guard_value* differs from
1457 *raw_value* (e.g. *guard_value* is the JSON-string-decoded form
1458 ``_coerced_for_guard`` produced purely to run the guards): an ordinary
1459 write, with no security-relevant transformation, must never be altered
1460 by this reconciliation step, or every other special-case handler
1461 downstream (checkbox coercion, the corrupted-value repair block) would
1462 see a different shape than it does today.
1464 Only when *guard_value* is a dict/list that DOES embed the sentinel
1465 does this return ``(effective, True)``, where ``effective`` is the
1466 MERGED container from ``_resolve_container_secret_write`` -- masked
1467 leaves restored to their stored secret, edited leaves as submitted --
1468 instead of *raw_value*, which still carries "[REDACTED]" on every
1469 untouched masked leaf and would otherwise be persisted verbatim.
1471 ``is_reconciled_container=True`` is the caller's signal to persist
1472 ``effective`` AS-IS, bypassing ``coerce_setting_for_write``. That
1473 coercion exists to turn a submitted SCALAR into the row's declared
1474 Python type (e.g. ``"5" -> 5`` for a number setting); every
1475 ui_element whose declared type is ``str`` -- "text", "textarea",
1476 "password", "select" -- would otherwise run a MERGED CONTAINER
1477 through ``str()``, flattening it to its Python repr and persisting a
1478 real, just-restored secret as a plain, non-redacted string under a
1479 key ``redact_value``/``is_sensitive_setting`` has no reason to treat
1480 as sensitive (#6201: this is exactly how the original fix for this
1481 function turned a data-loss bug into a disclosure one). A row's
1482 ``ui_element`` can disagree with its actual stored shape -- e.g. a
1483 legacy row or a ``PUT``-created key with explicit ``ui_element="text"``
1484 can hold a dict, even though new dict values now infer "json" -- so the
1485 submitted VALUE's shape, once reconciliation has happened, is what
1486 decides whether coercion is safe to run, not the row's declared type.
1487 ``coerce_setting_for_write``'s "json"/"multiselect" converters already
1488 pass a dict/list through unchanged, so skipping the call entirely for
1489 a reconciled container changes nothing for those two ui_elements --
1490 only "text"/"textarea"/"password"/"select" (whose converter is
1491 ``str``) and "number"/"checkbox"/"range"/"slider" (whose converter
1492 would silently discard the container's structure) are protected by
1493 this.
1494 """
1495 if not isinstance(guard_value, (dict, list)):
1496 return raw_value, False
1497 if not _container_embeds_sentinel(
1498 guard_value, DataSanitizer.REDACTION_TEXT
1499 ):
1500 return raw_value, False
1501 # NOTE (cross-helper coupling, deliberate but latent -- read before
1502 # editing either side). ``_is_rejected`` is DISCARDED here. When
1503 # ``_resolve_container_secret_write`` rejects -- an orphan sentinel
1504 # (a masked leaf with nothing in the stored row to restore it from), a
1505 # spliced "[REDACTED],x" leaf, a length-changed list, an untrusted
1506 # list position -- ``effective`` is the RAW submission, still carrying
1507 # the literal placeholder text, and this returns it with
1508 # ``is_reconciled_container=True``. Every caller reads that flag as
1509 # "persist as-is, skipping BOTH ``coerce_setting_for_write`` and
1510 # ``validate_setting``", so a rejected write would store "[REDACTED]"
1511 # as the credential with both checks bypassed.
1512 #
1513 # That is unreachable today, and only for one reason: all three call
1514 # sites run ``_embeds_redaction_sentinel`` FIRST, on the SAME
1515 # ``guard_value``, and it 400s on exactly the conditions that set
1516 # ``rejected`` here (it calls ``_resolve_container_secret_write`` with
1517 # the same arguments and returns its ``is_rejected``). By the time
1518 # this function runs, ``rejected`` is therefore always False. This is
1519 # the second undocumented coupling in this file (the first is
1520 # ``_merge_redacted_container``'s depth-cap NOTE) and is recorded
1521 # rather than removed because propagating the rejection would change
1522 # this helper's return arity at three call sites with three different
1523 # error conventions (a 400 JSONResponse, a per-key failed_count skip,
1524 # and a flash+302). If a caller is ever added that does NOT run
1525 # ``_embeds_redaction_sentinel`` on the same guard value first, or if
1526 # that guard's rejection conditions are narrowed, this becomes a live
1527 # fail-open and the rejection MUST be propagated instead.
1528 effective, _is_noop, _is_rejected = _resolve_container_secret_write(
1529 guard_value, stored_value, DataSanitizer.REDACTION_TEXT
1530 )
1531 return effective, True
1534def _is_secret_empty_noop(
1535 key: str,
1536 ui_element: str | None,
1537 value: object,
1538 stored_value: object = None,
1539) -> bool:
1540 """True when a secret write must be ignored: the redaction sentinel
1541 for any sensitive setting, or an empty string for password inputs
1542 (which render blank, so an untouched field must not wipe the secret).
1544 The ``ui_element == "password"`` narrowing matters:
1545 ``notifications.service_url`` is a sensitive setting on a ``textarea``
1546 that renders blank once redacted, but its blankness is advertised (the
1547 placeholder names the explicit clear gesture) rather than a stand-in
1548 for an unknown value, so an empty write there is a deliberate "clear
1549 it" gesture and must reach the database (#5960).
1551 A dict/list value goes through the container arm, delegated to
1552 ``_resolve_container_secret_write`` / ``_merge_redacted_container``:
1553 ``redact_value`` can mask a container's string leaves via
1554 ``_force_redact_strings`` or by SUB-KEY rather than replacing the whole
1555 value with the bare sentinel (see its docstring), so the exact-string
1556 check above never sees a container's round-tripped sentinel. Without
1557 this arm, a GET-then-save-untouched container -- e.g. ``{"uri":
1558 "[REDACTED]", "token": "[REDACTED]", "port": 19530}`` -- would sail past
1559 every guard here (all are ``isinstance(value, str)``-gated) and persist
1560 the sentinel over the real credential.
1562 The merge is per-LEAF, not all-or-nothing: every leaf that is exactly
1563 the sentinel is checked against ``stored_value`` at the same path, and
1564 every other leaf (an edited sibling, a non-string port/toggle) is kept
1565 as submitted. The write is a no-op only when that merge comes out
1566 byte-for-byte identical to what's already stored -- i.e. nothing about
1567 the container actually changed. A container that merges to something
1568 DIFFERENT from what's stored is a genuine edit (handled by
1569 ``_embeds_redaction_sentinel``'s reject path only if a leaf can't be
1570 resolved; otherwise it is written, see the update-route call sites,
1571 which persist the merged value rather than this function's raw
1572 argument). If the caller has no stored value to compare against
1573 (``stored_value is None``, e.g. no prior row), a sentinel-bearing
1574 container can never be verified as an untouched round-trip and is
1575 therefore never treated as this kind of no-op.
1577 The dict/list arm runs BEFORE the ``is_sensitive_setting`` gate below,
1578 unlike the string arm: ``redact_value`` recurses into a container by
1579 SUB-KEY, so a setting whose OUTER key is not itself sensitive (e.g.
1580 ``llm.model_kwargs``) can still have a sensitive-named leaf inside it
1581 (``stop_token``, ``page_token``) masked to the sentinel on the way out.
1582 Gating the container check on the outer key's sensitivity -- as this
1583 function used to -- means that sentinel round-trips straight back into
1584 the leaf on the very next save, since no guard ever looked at it. The
1585 literal string "[REDACTED]" is not a value any real setting leaf would
1586 hold, sensitive or not, so scanning every dict/list write for it
1587 unconditionally closes that gap without needing to re-derive which
1588 nested leaf name is "sensitive" here.
1589 """
1590 if isinstance(value, (dict, list)):
1591 _effective, is_noop, _is_rejected = _resolve_container_secret_write(
1592 value, stored_value, DataSanitizer.REDACTION_TEXT
1593 )
1594 return is_noop
1595 if not DataSanitizer.is_sensitive_setting(key, ui_element):
1596 return False
1597 if isinstance(value, str): 1597 ↛ 1601line 1597 didn't jump to line 1601 because the condition on line 1597 was always true
1598 return value == DataSanitizer.REDACTION_TEXT or (
1599 value == "" and ui_element == "password"
1600 )
1601 return False
1604def _redaction_sentinel_error(ui_element: str | None) -> str:
1605 """Explain the 400 raised for a value that embeds the redaction sentinel.
1607 Shared by every write route so the message is identical wherever it
1608 fires, but the "submit an empty value to clear it" hint only holds for
1609 non-password sensitive settings such as the ``notifications.service_url``
1610 textarea. ``_is_secret_empty_noop`` makes an empty write to a
1611 ``password`` input a deliberate no-op (those fields render blank, so an
1612 untouched form must not wipe the secret), and
1613 ``_embeds_redaction_sentinel`` gates on sensitivity alone, so a
1614 password-backed setting can reach this error through a direct API call
1615 even though the UI cannot produce it. Advertising the empty-value
1616 escape there would tell the caller to do something that provably
1617 cannot work, so the hint is dropped for password inputs.
1618 """
1619 message = (
1620 "Value contains the redaction placeholder "
1621 f"{DataSanitizer.REDACTION_TEXT!r}. The stored value is hidden, so "
1622 "it cannot be edited in place — retype the whole value"
1623 )
1624 if ui_element == "password":
1625 return (
1626 f"{message}. To clear a password setting, clear the source "
1627 "environment variable or use settings import."
1628 )
1629 return f"{message}, or submit an empty value to clear it."
1632def _embeds_redaction_sentinel(
1633 key: str,
1634 ui_element: str | None,
1635 value: object,
1636 stored_value: object = None,
1637) -> bool:
1638 """True when a submitted sensitive value *embeds* the redaction sentinel.
1640 ``_is_secret_empty_noop`` covers the exact sentinel: an untouched field
1641 round-tripped from a settings API read, which is benign and silently
1642 ignored. This covers the *edited* field. Password inputs render blank,
1643 so they cannot produce this, but ``notifications.service_url`` is a
1644 sensitive setting on a ``textarea`` (the first non-password sensitive
1645 setting in the codebase), and editing its comma-separated URL list is
1646 the normal workflow. A stale client that rendered the sentinel yields
1647 values like ``"[REDACTED],discord://webhook/tok"``, which is not an
1648 exact match and would otherwise persist verbatim and silently break
1649 every notification. No legitimate secret contains the sentinel, so this
1650 is a hard 400 rather than a no-op: unlike the exact-match case the user
1651 made an edit, and silently dropping it would look like a successful save.
1653 A dict/list value takes the container arm, delegated to
1654 ``_resolve_container_secret_write`` / ``_merge_redacted_container``.
1655 This is a REJECT only, never a silent partial write: a container is
1656 rejected when
1658 1. a leaf equals the sentinel exactly but ``stored_value`` is itself
1659 empty (a newly-introduced key, a structural mismatch, or no
1660 ``stored_value`` at all) -- there is nothing to "keep", so
1661 persisting it would write the literal placeholder text. This is
1662 only the emptiness half of the check ``redact_value`` masks on;
1663 the leaf's own sub-key sensitivity is never consulted here, so a
1664 sentinel restores from ANY non-empty stored value at that path,
1665 not only one that was actually masked (a stored ``{"port":
1666 19530}`` is never masked on read, but a submitted ``{"port":
1667 "[REDACTED]"}`` over it still restores ``19530``). Narrowing
1668 ``DEFAULT_SENSITIVE_KEYS`` / ``is_sensitive_setting`` therefore
1669 does not narrow what this merge will restore; or
1670 2. a leaf merely CONTAINS the sentinel as a substring without being
1671 an exact match (e.g. a stale client's rendered
1672 "[REDACTED],new-value") -- an edit, but a corrupted one that
1673 would splice the placeholder text into the stored value, same as
1674 the plain string case below.
1676 A container where every sentinel-bearing leaf DOES resolve against
1677 ``stored_value`` is NOT rejected here, even when other, non-masked
1678 leaves were edited or a masked leaf's non-string sibling changed:
1679 ``redact_value`` masks a container by SUB-KEY, so a MIXED container --
1680 e.g. ``{"api_key": "[REDACTED]", "description": "My engine",
1681 "requires_api_key": true}`` -- has only its sensitive leaf masked, and
1682 an untouched GET-then-PUT round-trip of it, or an edit to
1683 ``description``, must not become an unsatisfiable 400 the client can
1684 never construct a passing request for (round 5 required EVERY maskable
1685 leaf be exactly the sentinel, rejecting exactly this shape). Merging
1686 the resolvable case instead of rejecting it is what lets
1687 ``_is_secret_empty_noop`` recognize the true no-op and the
1688 update-route call sites persist the resolved edit -- see
1689 ``_reconciled_container_value_for_write``.
1691 The dict/list arm runs BEFORE the ``is_sensitive_setting`` gate below,
1692 for the same reason ``_is_secret_empty_noop`` does: ``redact_value``
1693 masks a container by SUB-KEY, so an outer key that is not itself
1694 sensitive (``llm.model_kwargs``) can still carry a masked leaf
1695 (``stop_token``) that this function must catch on write-back. No real
1696 setting leaf ever legitimately holds the literal string
1697 "[REDACTED]", so checking every dict/list write for it, regardless of
1698 the outer key, cannot reject a value nothing would ever submit.
1699 """
1700 if isinstance(value, (dict, list)):
1701 _effective, _is_noop, is_rejected = _resolve_container_secret_write(
1702 value, stored_value, DataSanitizer.REDACTION_TEXT
1703 )
1704 return is_rejected
1705 if not DataSanitizer.is_sensitive_setting(key, ui_element):
1706 return False
1707 if isinstance(value, str):
1708 return (
1709 DataSanitizer.REDACTION_TEXT in value
1710 and value != DataSanitizer.REDACTION_TEXT
1711 )
1712 return False
1715def _embeds_sentinel_on_create(
1716 key: str, ui_element: object, value: object
1717) -> bool:
1718 """Sentinel check for the CREATE paths.
1720 Creation has no prior value, so the sentinel cannot mean "keep the
1721 stored secret" the way it does on the update path — every occurrence
1722 of it, exact match included, is a corrupted client value that would be
1723 stored verbatim as the credential. A dict/list value gets the same
1724 treatment via ``_container_embeds_sentinel``: there is no "pure
1725 round-trip" exemption here (unlike ``_is_secret_empty_noop`` on the
1726 update path) because creation has no prior stored value for an untouched
1727 field to round-trip from.
1729 The dict/list check runs regardless of the outer key's sensitivity, for
1730 the same reason as the update-path guards: a container's sensitive leaf
1731 can be named differently than its outer key, and no legitimate value
1732 ever contains the literal sentinel string.
1733 """
1734 if isinstance(value, (dict, list)):
1735 return _container_embeds_sentinel(value, DataSanitizer.REDACTION_TEXT)
1736 if not DataSanitizer.is_sensitive_setting(
1737 key, ui_element if isinstance(ui_element, str) else None
1738 ):
1739 return False
1740 if isinstance(value, str):
1741 return DataSanitizer.REDACTION_TEXT in value
1742 return False
1745def _is_allowed_new_setting_key(key: str) -> bool:
1746 """Return True if *key* is permitted to be created via the web API."""
1747 # Reject malformed keys (blank, trailing/leading dot, empty ".." segment,
1748 # stray whitespace) before the namespace check — a trailing-dot key such
1749 # as ``local_search_chunk_size.`` otherwise passes the prefix allow-list
1750 # and corrupts prefix lookups (see #4840).
1751 if not is_valid_setting_key(key):
1752 return False
1753 key = key.lower()
1754 for prefix in BLOCKED_SETTING_PREFIXES:
1755 if key.startswith(prefix):
1756 return False
1757 for prefix in ALLOWED_SETTING_PREFIXES:
1758 if key.startswith(prefix):
1759 return True
1760 return False
1763def _new_key_rejection_reason(key) -> str:
1764 """Explain why ``_is_allowed_new_setting_key`` rejected *key*.
1766 Distinguishes a malformed key (bad syntax) from an allowed-namespace
1767 violation so an API consumer with, say, a trailing-dot key (#4840) is
1768 pointed at the real problem instead of being told it's a namespace issue.
1769 """
1770 if not is_valid_setting_key(key):
1771 return f"Setting key is malformed: {key!r}"
1772 return f"Creating settings under this namespace is not allowed: {key}"
1775@router.get("/")
1776def settings_page(
1777 request: Request, username: Annotated[str, Depends(require_auth)]
1778):
1779 """Main settings dashboard with links to specialized config pages"""
1780 return templates.TemplateResponse(
1781 request=request,
1782 name="settings_dashboard.html",
1783 context={"request": request},
1784 )
1787def _save_all_settings_sync(form_data: dict, username: str):
1788 """Synchronous body of save_all_settings.
1790 Separated so the `async def` handler can offload the entire
1791 SQLAlchemy + validation + re-read block to a thread via
1792 asyncio.to_thread, freeing the event loop during a bulk save.
1793 """
1794 try:
1795 from ...security.data_sanitizer import DataSanitizer
1797 with get_user_db_session(username) as db_session:
1798 settings_manager = get_settings_manager(db_session, username)
1799 if settings_manager.settings_locked:
1800 return JSONResponse(
1801 {
1802 "status": "error",
1803 "message": "Settings are locked and cannot be changed",
1804 },
1805 status_code=403,
1806 )
1808 # Track validation errors
1809 validation_errors = []
1810 settings_by_type: dict[str, Any] = {}
1812 # Track changes for logging
1813 updated_settings = []
1814 created_settings = []
1816 # Store original values for better messaging
1817 original_values = {}
1819 # Fetch all settings and remove non-editable keys
1820 all_db_settings = _filter_editable_settings(form_data, db_session)
1822 # Reject public hostnames being added to the local-hosts allowlist,
1823 # and inherently-public engines being added to the trusted-engines
1824 # list.
1825 _hosts_err = first_egress_validation_error(
1826 form_data, all_db_settings
1827 )
1828 if _hosts_err is not None:
1829 validation_errors.append(_hosts_err)
1830 return JSONResponse(
1831 {
1832 "status": "error",
1833 "message": "Validation errors",
1834 "errors": validation_errors,
1835 },
1836 status_code=400,
1837 )
1839 # Update each setting
1840 for key, value in form_data.items():
1841 # Skip corrupted keys or empty strings as keys
1842 if not key or not isinstance(key, str) or key.strip() == "":
1843 continue
1845 # Get the setting metadata from pre-fetched dict
1846 current_setting = all_db_settings.get(key)
1848 if key == "search.fetch.summary_input_max_length":
1849 try:
1850 value = to_positive_int(value)
1851 except ValueError:
1852 validation_errors.append(
1853 {
1854 "key": key,
1855 "name": "Fetch Summary Input Max Length",
1856 "error": "Value must be a positive integer",
1857 }
1858 )
1859 continue
1861 # The sentinel guards below must see a JSON-STRING-encoded
1862 # container coerced to its real dict/list shape, or a
1863 # masked container submitted in that form (a supported
1864 # input shape for a "json" ui_element) sails past the
1865 # container arm entirely -- see ``_coerced_for_guard``.
1866 _guard_value = (
1867 _coerced_for_guard(
1868 current_setting.key, current_setting.ui_element, value
1869 )
1870 if current_setting
1871 else value
1872 )
1874 # SAFETY NET: the redaction sentinel is never a "clear the
1875 # value" request, and neither is an empty string on a
1876 # password input. Reasons:
1877 # 1. The form templates (Jinja2 + JS-rendered) deliberately
1878 # render password inputs empty so the saved value never
1879 # enters the HTML source. A user who blurs the field
1880 # without typing must not wipe their stored API key.
1881 # 2. /settings/api redacts secret values to the redaction
1882 # sentinel ("[REDACTED]"). A stale browser tab could
1883 # submit it back — that must be idempotent on the DB, or
1884 # the round-trip would persist the literal sentinel over
1885 # the real secret.
1886 # 3. Defense-in-depth against direct cURL or automation
1887 # mistakes that POST an empty/sentinel value.
1888 # The empty-string half is narrowed to password inputs:
1889 # non-password sensitive controls (the
1890 # notifications.service_url textarea) render their real value
1891 # and must stay clearable (#5960). To unset a password
1892 # setting, clear the source env var or use settings import.
1893 if current_setting and _is_secret_empty_noop(
1894 key,
1895 current_setting.ui_element,
1896 _guard_value,
1897 current_setting.value,
1898 ):
1899 logger.debug(
1900 f"Skipping empty secret write for {key} (no-op)"
1901 )
1902 continue
1904 # A value that merely *embeds* the sentinel is a corrupted
1905 # edit, not an untouched round-trip: reject it loudly instead
1906 # of persisting a secret with "[REDACTED]" spliced into it
1907 # (#5947).
1908 if current_setting and _embeds_redaction_sentinel(
1909 key,
1910 current_setting.ui_element,
1911 _guard_value,
1912 current_setting.value,
1913 ):
1914 logger.warning(
1915 "Rejected redaction-sentinel value for {!r} via "
1916 "save_all_settings (user={!r})",
1917 key,
1918 username,
1919 )
1920 validation_errors.append(
1921 {
1922 "key": key,
1923 "name": current_setting.name,
1924 "error": _container_rejection_error(
1925 current_setting.ui_element, _guard_value
1926 ),
1927 }
1928 )
1929 continue
1931 # EARLY VALIDATION: Convert checkbox values BEFORE any other processing
1932 # This prevents incorrect triggering of corrupted value detection
1933 if current_setting and current_setting.ui_element == "checkbox":
1934 if not isinstance(value, bool):
1935 # Never interpolate a settings VALUE into a log
1936 # line -- same rule as ``manager.py``'s converter
1937 # (#6201). A checkbox row can hold an arbitrary
1938 # submitted value here (that is exactly why this
1939 # branch exists), and a stale client or a
1940 # mis-registered ui_element can put a credential or
1941 # a secret-bearing container in it. The key, the
1942 # type name and the length identify the bad
1943 # submission without disclosing it.
1944 logger.debug(
1945 "Converting checkbox {} from {} (len={}) to bool",
1946 key,
1947 type(value).__name__,
1948 len(value)
1949 if isinstance(value, (str, list, dict, tuple))
1950 else "n/a",
1951 )
1952 value = parse_boolean(value)
1953 form_data[key] = (
1954 value # Update the form_data with converted value
1955 )
1957 # Store original value for messaging
1958 if current_setting:
1959 original_values[key] = current_setting.value
1961 # Determine setting type and category
1962 if key.startswith("llm."):
1963 setting_type = SettingType.LLM
1964 category = "llm_general"
1965 if (
1966 "temperature" in key
1967 or "max_tokens" in key
1968 or "batch" in key
1969 or "layers" in key
1970 ):
1971 category = "llm_parameters"
1972 elif key.startswith("search."):
1973 setting_type = SettingType.SEARCH
1974 category = "search_general"
1975 if (
1976 "iterations" in key
1977 or "results" in key
1978 or "region" in key
1979 or "questions" in key
1980 or "section" in key
1981 ):
1982 category = "search_parameters"
1983 elif key.startswith("report."):
1984 setting_type = SettingType.REPORT
1985 category = "report_parameters"
1986 elif key.startswith("database."):
1987 setting_type = SettingType.DATABASE
1988 category = "database_parameters"
1989 elif key.startswith("app."):
1990 setting_type = SettingType.APP
1991 category = "app_interface"
1992 elif key.startswith("chat."): 1992 ↛ 1993line 1992 didn't jump to line 1993 because the condition on line 1992 was never true
1993 setting_type = SettingType.CHAT
1994 category = "chat"
1995 else:
1996 setting_type = None
1997 category = None
1999 # Special handling for corrupted or empty values
2000 if value == "[object Object]" or (
2001 isinstance(value, str)
2002 and value.strip() in ["{}", "[]", "{", "["]
2003 ):
2004 if key.startswith("report."):
2005 value = {}
2006 else:
2007 # Use default or null for other types
2008 if key == "llm.model":
2009 # Repair defaults must match main (#3348): an empty
2010 # model lets the provider pick, "ollama" keeps a
2011 # local-only install local instead of silently
2012 # switching it to a cloud provider, and "auto" is
2013 # NOT a registered engine — the factory fails
2014 # closed on it, so a repaired install could no
2015 # longer search at all.
2016 value = ""
2017 elif key == "llm.provider":
2018 value = "ollama"
2019 elif key == "search.tool":
2020 value = DEFAULT_SEARCH_TOOL
2021 elif key in ["app.theme", "app.default_theme"]:
2022 # Must stay a value the theme registry actually
2023 # serves; "dark" was reset here for a long time
2024 # after the registry stopped having it.
2025 value = "system"
2026 else:
2027 value = None
2029 logger.warning(
2030 "Corrected corrupted value for {}: {}",
2031 _log_value_preview(key),
2032 value,
2033 )
2034 # NOTE: No JSON pre-parsing is done here. After the
2035 # corruption replacement above, values are Python dicts
2036 # (e.g. {}), hardcoded strings, or None — none are JSON
2037 # strings that need parsing. Type conversion below via
2038 # coerce_setting_for_write() handles everything; that
2039 # function delegates to get_typed_setting_value() which
2040 # already parses JSON internally for "json" and
2041 # "multiselect" ui_elements.
2043 if current_setting:
2044 # If the guards above let this through because a masked
2045 # container's sentinel leaves resolved against the
2046 # stored row (see ``_reconciled_container_value_for_write``),
2047 # persist the MERGED value -- secrets restored, edits
2048 # applied -- not the raw submission, which still carries
2049 # "[REDACTED]" on every untouched masked leaf. An
2050 # ordinary write (no sentinel involved) is returned
2051 # unchanged, so this has no effect on it.
2052 value, _is_reconciled_container = (
2053 _reconciled_container_value_for_write(
2054 value, _guard_value, current_setting.value
2055 )
2056 )
2058 # Coerce to correct Python type (e.g. str "5" → int 5
2059 # for number settings, str "true" → bool for checkboxes).
2060 # A reconciled secret-bearing container is persisted
2061 # exactly as merged instead: coercing it through a
2062 # str-typed ui_element ("text"/"textarea"/"select")
2063 # would flatten it to its Python repr, persisting a
2064 # just-restored secret as a plain string (#6201). See
2065 # ``_reconciled_container_value_for_write``'s docstring.
2066 converted_value = (
2067 value
2068 if _is_reconciled_container
2069 else coerce_setting_for_write(
2070 key=current_setting.key,
2071 value=value,
2072 ui_element=current_setting.ui_element,
2073 )
2074 )
2076 # Validate the setting. A reconciled secret-bearing
2077 # container must never reach ``validate_setting``, and
2078 # the load-bearing reason is TYPE REJECTION rather than
2079 # logging: for a "number"/"range" row the value flows
2080 # into ``get_typed_setting_value``'s bare ``float(x)``,
2081 # which fails, so the row is reported "Value must be a
2082 # number" and the merged edit -- with the secret just
2083 # restored into it -- is lost. That same converter also
2084 # used to log the raw value at WARNING; that half is
2085 # closed at the converter as of #6201 round 6 and no
2086 # longer justifies the skip on its own (see
2087 # ``_coerced_for_guard``, whose comment says the same).
2088 # Only a user-constructed row (dict/list value on a
2089 # non-container ui_element) can reach this; skipping
2090 # validation is safe for the secret itself because the
2091 # merge already resolved the container against the
2092 # stored row. ACCEPTED LOSS: it does also skip the
2093 # options-membership / min-max / checkbox-type checks
2094 # for such a row, so e.g. a "select" row already
2095 # holding a container accepts a merged container with
2096 # no options check.
2097 is_valid, error_message = (
2098 (True, None)
2099 if _is_reconciled_container
2100 else validate_setting(current_setting, converted_value)
2101 )
2103 if is_valid:
2104 # Save WITHOUT committing — one final commit runs
2105 # below after the validation pass, so a later
2106 # validation error rolls back every preceding
2107 # write instead of leaving the DB half-saved.
2108 success = set_setting(
2109 key,
2110 converted_value,
2111 commit=False,
2112 db_session=db_session,
2113 )
2114 if success:
2115 updated_settings.append(key)
2116 else:
2117 db_session.rollback()
2118 logger.error("Failed to persist setting {}", key)
2119 return JSONResponse(
2120 {
2121 "status": "error",
2122 "message": "Failed to save setting.",
2123 },
2124 status_code=500,
2125 )
2127 # Track settings by type for exporting
2128 if current_setting.type not in settings_by_type:
2129 settings_by_type[current_setting.type] = []
2130 settings_by_type[current_setting.type].append(
2131 current_setting
2132 )
2133 else:
2134 # Add to validation errors
2135 validation_errors.append(
2136 {
2137 "key": key,
2138 "name": current_setting.name,
2139 "error": error_message,
2140 }
2141 )
2142 else:
2143 # Namespace validation: reject new keys outside allowed
2144 # prefixes (existing keys above bypass — updates only).
2145 if not _is_allowed_new_setting_key(key):
2146 logger.warning(
2147 "Security: Rejected setting outside allowed "
2148 "namespaces: {} (user={!r})",
2149 _log_value_preview(key),
2150 username,
2151 )
2152 validation_errors.append(
2153 {
2154 "key": key,
2155 "name": key,
2156 "error": _new_key_rejection_reason(key),
2157 }
2158 )
2159 continue
2161 # Creation has no prior value, so the sentinel cannot
2162 # mean "keep the stored secret" — every occurrence of it,
2163 # exact match included, would be stored verbatim as the
2164 # credential. ui_element is not yet known here, so
2165 # sensitivity is decided by the key's leaf name (#5947).
2166 if _embeds_sentinel_on_create(key, None, value):
2167 logger.warning(
2168 "Rejected redaction-sentinel value for new key "
2169 "{} via save_all_settings (user={!r})",
2170 _log_value_preview(key),
2171 username,
2172 )
2173 validation_errors.append(
2174 {
2175 "key": key,
2176 "name": key.split(".")[-1]
2177 .replace("_", " ")
2178 .title(),
2179 "error": _container_rejection_error(
2180 None, value
2181 ),
2182 }
2183 )
2184 continue
2186 # Create a new setting
2187 new_setting = {
2188 "key": key,
2189 "value": value,
2190 "type": setting_type.value.lower()
2191 if setting_type is not None
2192 else "app",
2193 "name": key.split(".")[-1].replace("_", " ").title(),
2194 "description": f"Setting for {key}",
2195 "category": category,
2196 "ui_element": "text", # Default UI element
2197 }
2199 # Determine better UI element based on value type
2200 if isinstance(value, bool):
2201 new_setting["ui_element"] = "checkbox"
2202 elif isinstance(value, (int, float)) and not isinstance(
2203 value, bool
2204 ):
2205 new_setting["ui_element"] = "number"
2206 elif isinstance(value, (dict, list)):
2207 new_setting["ui_element"] = "textarea"
2209 # A JSON body can carry NaN, Infinity or an integer too
2210 # large for a float, which the number type rejects on
2211 # read.
2212 rejection = _strict_read_rejection(
2213 key, value, new_setting["ui_element"]
2214 )
2215 if rejection is not None:
2216 validation_errors.append(
2217 {
2218 "key": key,
2219 "name": new_setting["name"],
2220 "error": rejection,
2221 }
2222 )
2223 continue
2225 # Create the setting without committing yet — see above
2226 db_setting = create_or_update_setting(
2227 new_setting, commit=False, db_session=db_session
2228 )
2230 if db_setting:
2231 created_settings.append(key)
2232 # Track settings by type for exporting
2233 if db_setting.type not in settings_by_type: 2233 ↛ 2235line 2233 didn't jump to line 2235 because the condition on line 2233 was always true
2234 settings_by_type[db_setting.type] = []
2235 settings_by_type[db_setting.type].append(db_setting)
2236 else:
2237 validation_errors.append(
2238 {
2239 "key": key,
2240 "name": new_setting["name"],
2241 "error": "Failed to create setting",
2242 }
2243 )
2245 # Report validation errors if any — roll back the whole batch
2246 # so a partial save isn't visible to the next request.
2247 if validation_errors:
2248 db_session.rollback()
2249 return JSONResponse(
2250 {
2251 "status": "error",
2252 "message": "Validation errors",
2253 "errors": validation_errors,
2254 },
2255 status_code=400,
2256 )
2258 # Prepare the effective response before committing. An invalid
2259 # environment override must refuse the batch before any edit or
2260 # post-commit effect becomes visible.
2261 all_settings = {}
2262 for setting in db_session.query(Setting).all():
2263 # Convert enum to string if present
2264 setting_type = setting.type
2265 if hasattr(setting_type, "value"):
2266 setting_type = setting_type.value
2268 all_settings[setting.key] = {
2269 "value": setting.value,
2270 "name": setting.name,
2271 "description": setting.description,
2272 "type": setting_type,
2273 "category": setting.category,
2274 "ui_element": setting.ui_element,
2275 "editable": setting.editable,
2276 "options": setting.options,
2277 "visible": setting.visible,
2278 "min_value": setting.min_value,
2279 "max_value": setting.max_value,
2280 "step": setting.step,
2281 }
2283 # Overlay operator-locked (LDR_*) settings onto the echo, the
2284 # way SettingsManager.get_all_settings() already does for
2285 # GET /settings/api. The settings form re-renders from this
2286 # payload, so without the overlay an env-pinned field comes back
2287 # showing the stale DB value and marked editable until a full
2288 # page reload (#5978). Port of main's
2289 # _shape_effective_setting_metadata.
2290 for _echo_key, _echo_meta in all_settings.items():
2291 if is_operator_only_setting(_echo_key):
2292 _echo_meta["editable"] = False
2293 if check_env_setting(_echo_key) is None:
2294 continue
2295 _echo_meta["value"] = get_typed_setting_value(
2296 key=_echo_key,
2297 value=_echo_meta.get("value"),
2298 ui_element=str(_echo_meta.get("ui_element", "text")),
2299 default=_echo_meta.get("value"),
2300 check_env=True,
2301 )
2302 _echo_meta["editable"] = False
2304 # The echo above covers only keys with a saved row. An override on
2305 # a declared (default) key without a row is part of the same
2306 # effective configuration, so it must refuse the batch too rather
2307 # than surface only on the next typed read.
2308 _defaults = settings_manager.default_settings
2309 for _default_key, _default_meta in _defaults.items():
2310 if (
2311 _default_key in all_settings
2312 or check_env_setting(_default_key) is None
2313 ):
2314 continue
2315 get_typed_setting_value(
2316 key=_default_key,
2317 value=_default_meta.get("value"),
2318 ui_element=str(_default_meta.get("ui_element", "text")),
2319 default=_default_meta.get("value"),
2320 check_env=True,
2321 )
2323 # Operator-gate the "unprotected" escape hatch out of the
2324 # egress-scope select's options unless explicitly enabled, and
2325 # normalise the displayed value (#5148 / 87537d9ec). Without
2326 # this, a save-and-refresh cycle would re-render the dropdown
2327 # from this echoed payload with the escape hatch back on offer.
2328 if "policy.egress_scope" in all_settings:
2329 all_settings["policy.egress_scope"] = (
2330 _shape_egress_scope_setting(
2331 "policy.egress_scope",
2332 all_settings["policy.egress_scope"],
2333 )
2334 )
2336 # Hide the operator-gated unencrypted "filesystem" PDF-storage
2337 # option from a save-and-refresh echo, same rationale as the
2338 # egress-scope shaping above (#5148 / fb49985aa).
2339 if "research_library.pdf_storage_mode" in all_settings:
2340 all_settings["research_library.pdf_storage_mode"] = (
2341 _shape_pdf_storage_mode_setting(
2342 "research_library.pdf_storage_mode",
2343 all_settings["research_library.pdf_storage_mode"],
2344 )
2345 )
2347 # Customize the success message based on what changed
2348 success_message = ""
2349 if len(updated_settings) == 1:
2350 # For a single update, provide more specific info about what changed
2351 key = updated_settings[0]
2352 # Reuse the already-fetched setting from our pre-fetched dict
2353 updated_setting = all_db_settings.get(key)
2354 name = (
2355 updated_setting.name
2356 if updated_setting
2357 else key.split(".")[-1].replace("_", " ").title()
2358 )
2360 # Format the message
2361 if key in original_values: 2361 ↛ 2379line 2361 didn't jump to line 2379 because the condition on line 2361 was always true
2362 # Get original value but comment out if not used
2363 # old_value = original_values[key]
2364 new_value = (
2365 updated_setting.value if updated_setting else None
2366 )
2368 # If it's a boolean, use "enabled/disabled" language
2369 if isinstance(new_value, bool):
2370 state = "enabled" if new_value else "disabled"
2371 success_message = f"{name} {state}"
2372 else:
2373 # For non-boolean values
2374 if isinstance(new_value, (dict, list)):
2375 success_message = f"{name} updated"
2376 else:
2377 success_message = f"{name} updated"
2378 else:
2379 success_message = f"{name} updated"
2380 else:
2381 # Multiple settings or generic message
2382 success_message = f"Settings saved successfully ({len(updated_settings)} updated, {len(created_settings)} created)"
2384 # Check if any warning-affecting settings were changed and include
2385 # warnings. Redact secret values in the echoed settings so a POST
2386 # response never ships plaintext API keys back to the browser —
2387 # matching the redaction the GET /settings/api endpoint applies.
2388 response_data = {
2389 "status": "success",
2390 "message": success_message,
2391 "updated": updated_settings,
2392 "created": created_settings,
2393 "settings": DataSanitizer.redact_settings_snapshot(
2394 all_settings
2395 ),
2396 }
2398 # The writes and response both validated: commit atomically.
2399 db_session.commit()
2400 changed_settings = updated_settings + created_settings
2401 # The batch is committed from here on. A settings error raised by
2402 # a post-commit effect must not be reported as a refused save.
2403 try:
2404 if changed_settings:
2405 settings_manager.emit_settings_changed_after_commit(
2406 changed_settings
2407 )
2409 invalidate_settings_caches(username)
2410 reschedule_document_jobs_if_needed(username, changed_settings)
2411 reschedule_zotero_jobs_if_needed(username, changed_settings)
2413 # Check if any warning-affecting settings were changed
2414 if any(
2415 key in WARNING_AFFECTING_KEYS for key in changed_settings
2416 ):
2417 warnings = calculate_warnings(username=username)
2418 response_data["warnings"] = warnings
2419 logger.info(
2420 f"Bulk settings update affected warning keys, calculated {len(warnings)} warnings"
2421 )
2422 except SettingsValueError as error:
2423 _log_post_save_settings_error(error)
2425 return response_data
2427 except SettingsValueError as error:
2428 return _settings_value_error_response(
2429 "settings for bulk save", error, bulk_save=True
2430 )
2431 except Exception:
2432 logger.exception("Error saving settings")
2433 return JSONResponse(
2434 {
2435 "status": "error",
2436 "message": "An internal error occurred while saving settings.",
2437 },
2438 status_code=500,
2439 )
2442@router.post("/save_all_settings")
2443@settings_limit
2444async def save_all_settings(
2445 request: Request,
2446 username: Annotated[str, Depends(require_auth)],
2447):
2448 """Handle saving all settings at once from the unified settings page.
2450 Thin async wrapper that reads the JSON body, then offloads the
2451 entire sync save to a thread so we don't block the event loop.
2452 """
2453 form_data, body_error = await read_json_dict(
2454 request, "status", "No settings data provided"
2455 )
2456 if body_error is not None:
2457 return body_error
2458 if not form_data:
2459 return JSONResponse(
2460 {"status": "error", "message": "No settings data provided"},
2461 status_code=400,
2462 )
2463 return await run_db_sync(_save_all_settings_sync, form_data, username)
2466@router.post("/reset_to_defaults")
2467@settings_limit
2468def reset_to_defaults(
2469 request: Request,
2470 username: Annotated[str, Depends(require_auth)],
2471):
2472 """Reset all settings to their default values.
2474 Preserves API keys and other password-type settings — overwriting
2475 these with empty defaults from the bundled JSON would silently
2476 erase the user's credentials and force them to re-enter every key.
2477 """
2478 try:
2479 with get_user_db_session(username) as db_session:
2480 settings_manager = get_settings_manager(db_session, username)
2482 # Settings lock (app.lock_settings) -- ported from main's
2483 # reset_to_defaults (#5659, "enforce the settings lock on delete,
2484 # import and reset"). SettingsManager already refuses when
2485 # locked, so the write cannot happen either way; this repeats
2486 # the check at the route so a locked instance answers 403
2487 # rather than 200 with nothing written, which is what main's
2488 # own comment gives as the reason. Without it the merge that
2489 # brought #5659 in would have silently dropped the fix, since
2490 # it landed in a Flask file this migration deletes.
2491 if settings_manager.settings_locked:
2492 return JSONResponse(
2493 {
2494 "status": "error",
2495 "message": "Settings are locked and cannot be reset",
2496 },
2497 status_code=403,
2498 )
2500 # Snapshot password/api-key settings before reset so we can
2501 # restore them. We use the DB row's ui_element to detect
2502 # password fields rather than a name pattern.
2503 preserved: dict[str, Any] = {}
2504 password_rows = (
2505 db_session.query(Setting)
2506 .filter(Setting.ui_element == "password")
2507 .all()
2508 )
2509 for row in password_rows:
2510 if row.value not in (None, ""):
2511 preserved[row.key] = row.value
2513 # preserve_environment_locked=True is load-bearing and was lost in
2514 # the port: it defaults to False, and import_settings writes rows
2515 # in BULK rather than through the _is_environment_locked-guarded
2516 # setters, so without it a reset overwrites the stored rows of
2517 # settings an operator has locked via LDR_* env vars — and drops
2518 # the policy_audit warning that records the attempt. The damage is
2519 # latent while the env var is set (reads prefer env) and surfaces
2520 # the moment the operator removes it.
2521 settings_manager.load_from_defaults_file(
2522 preserve_environment_locked=True
2523 )
2524 reset_setting_keys = tuple(settings_manager.default_settings)
2526 # Restore preserved values
2527 for key, value in preserved.items():
2528 try:
2529 settings_manager.set_setting(key, value, commit=False)
2530 except Exception:
2531 logger.exception(
2532 f"Could not restore preserved setting {key}"
2533 )
2534 if preserved:
2535 db_session.commit()
2537 logger.info(
2538 "Successfully reset settings to defaults "
2539 f"(preserved {len(preserved)} password-type setting(s))"
2540 )
2541 invalidate_settings_caches(username)
2542 reschedule_document_jobs_if_needed(username, reset_setting_keys)
2543 reschedule_zotero_jobs_if_needed(username, reset_setting_keys)
2545 except Exception:
2546 logger.exception("Error importing default settings")
2547 return JSONResponse(
2548 {
2549 "status": "error",
2550 "message": "Failed to reset settings to defaults",
2551 },
2552 status_code=500,
2553 )
2555 return {
2556 "status": "success",
2557 "message": "All settings have been reset to default values",
2558 }
2561def _save_settings_sync(form_data: dict, username: str) -> dict:
2562 """Sync helper for save_settings — runs the bulk setting writes.
2564 Returns an outcome dict so the caller can flash user-visible feedback
2565 on the no-JS fallback path (``ok``, ``policy_error``, ``failed``,
2566 ``rejected``) — Flask's flash()-based feedback was otherwise lost in
2567 the migration, leaving the no-JS form a silent redirect regardless of
2568 success, partial rejection, or failure.
2569 """
2570 with get_user_db_session(username) as db_session:
2571 settings_manager = get_settings_manager(db_session, username)
2573 # Fetch all settings and remove non-editable keys
2574 all_db_settings = _filter_editable_settings(form_data, db_session)
2576 # Egress-policy validators — the JSON route (save_all_settings) runs
2577 # these; the POST fallback must too, or a JS-disabled client could
2578 # whitelist a public hostname as "local", which the JSON route does
2579 # not permit.
2580 _policy_err = first_egress_validation_error(form_data, all_db_settings)
2581 if _policy_err is not None: 2581 ↛ 2582line 2581 didn't jump to line 2582 because the condition on line 2581 was never true
2582 logger.warning(
2583 "Rejected settings POST: {}",
2584 _log_value_preview(
2585 _policy_err.get("error", "Invalid policy setting")
2586 ),
2587 )
2588 return {
2589 "ok": False,
2590 "policy_error": _policy_err.get(
2591 "error", "Invalid policy setting"
2592 ),
2593 "failed": 0,
2594 "rejected": 0,
2595 }
2597 failed_count = 0
2598 rejected_count = 0
2599 changed_settings: list[str] = []
2600 sentinel_rejected_ui_elements: list[str | None] = []
2601 # Parallel to sentinel_rejected_ui_elements: whether that specific
2602 # rejection was the depth cap rather than an actual sentinel, so the
2603 # aggregated flash message below doesn't misreport a too-deep
2604 # payload as "contains the redaction placeholder" (see
2605 # ``_container_too_deep_error``).
2606 sentinel_rejected_too_deep: list[bool] = []
2607 for key, value in form_data.items():
2608 try:
2609 db_setting = all_db_settings.get(key)
2611 if key == "search.fetch.summary_input_max_length":
2612 try:
2613 value = to_positive_int(value)
2614 except ValueError:
2615 failed_count += 1
2616 continue
2618 # Namespace validation: reject new keys outside allowed
2619 # prefixes. Existing keys (updates) bypass this check — it
2620 # only applies to creation of brand-new rows through this
2621 # form-POST route.
2622 if db_setting is None and not _is_allowed_new_setting_key(key):
2623 logger.warning(
2624 "Security: Rejected setting outside allowed "
2625 "namespaces: {} (user={!r})",
2626 _log_value_preview(key),
2627 username,
2628 )
2629 rejected_count += 1
2630 continue
2632 # Creation has no prior value, so the sentinel cannot mean
2633 # "keep the stored secret" — every occurrence is a
2634 # corrupted client value that would be stored verbatim as
2635 # the credential. This is the one write path that never
2636 # had this check wired in (#5947 added it to
2637 # save_all_settings and api_update_setting, but not here);
2638 # ui_element is unknown at creation time, so sensitivity is
2639 # decided by the key's leaf name alone, same as the other
2640 # create paths.
2641 if db_setting is None and _embeds_sentinel_on_create(
2642 key, None, value
2643 ):
2644 logger.warning(
2645 "Rejected redaction-sentinel value for new key "
2646 "{} via save_settings (user={!r})",
2647 _log_value_preview(key),
2648 username,
2649 )
2650 sentinel_rejected_ui_elements.append(None)
2651 sentinel_rejected_too_deep.append(
2652 isinstance(value, (dict, list))
2653 and _container_exceeds_max_depth(value)
2654 )
2655 continue
2657 # See ``_coerced_for_guard``: the sentinel guards must see a
2658 # JSON-string-encoded container coerced to its real
2659 # dict/list shape, or a masked container submitted in that
2660 # form sails past the container arm entirely.
2661 _guard_value = (
2662 _coerced_for_guard(
2663 db_setting.key, db_setting.ui_element, value
2664 )
2665 if db_setting
2666 else value
2667 )
2669 # SAFETY NET: the redaction sentinel is a no-op for every
2670 # sensitive setting, and an empty string is a no-op only for
2671 # password inputs — never "clear my key". The no-JS form
2672 # renders password inputs empty (and GET redacts them to the
2673 # sentinel), so a plain form submit must not wipe the stored
2674 # secret. Non-password sensitive controls (the
2675 # notifications.service_url textarea) stay clearable (#5960).
2676 # Matches the guards in save_all_settings +
2677 # api_update_setting.
2678 if db_setting and _is_secret_empty_noop(
2679 key, db_setting.ui_element, _guard_value, db_setting.value
2680 ):
2681 logger.debug(
2682 f"Skipping empty secret write for {key} via "
2683 "save_settings (no-op)"
2684 )
2685 continue
2687 # An embedded sentinel is a corrupted edit, not a round-trip.
2688 # This route reports per-setting outcomes by flash message,
2689 # so collect it rather than saving a secret with "[REDACTED]"
2690 # spliced into it (#5947).
2691 if db_setting and _embeds_redaction_sentinel(
2692 key, db_setting.ui_element, _guard_value, db_setting.value
2693 ):
2694 logger.warning(
2695 "Rejected redaction-sentinel value for {!r} via "
2696 "save_settings (user={!r})",
2697 key,
2698 username,
2699 )
2700 sentinel_rejected_ui_elements.append(db_setting.ui_element)
2701 sentinel_rejected_too_deep.append(
2702 isinstance(_guard_value, (dict, list))
2703 and _container_exceeds_max_depth(_guard_value)
2704 )
2705 continue
2707 if db_setting:
2708 # An HTML form cannot express None. A <select> whose
2709 # options include a null value (e.g. "All Time" for
2710 # search.engine.web.serper.default_params.time_period)
2711 # posts back "" for that choice, and "" is not in the
2712 # allowed values, so validate_setting below rejects a
2713 # value the user never changed. The JSON route is
2714 # unaffected because it sends a real null.
2715 #
2716 # Only "" is mapped, and only when null is genuinely an
2717 # allowed option — so this cannot turn a typo into a
2718 # silent null on a select that does not permit one.
2719 if (
2720 value == ""
2721 and db_setting.ui_element == "select"
2722 and db_setting.options
2723 and any(
2724 (opt.get("value") if isinstance(opt, dict) else opt)
2725 is None
2726 for opt in list(db_setting.options)
2727 )
2728 ):
2729 value = None
2731 # Persist the MERGED value when a masked container's
2732 # sentinel leaves resolved against the stored row
2733 # (see ``_reconciled_container_value_for_write``); an
2734 # ordinary write is returned unchanged.
2735 value, _is_reconciled_container = (
2736 _reconciled_container_value_for_write(
2737 value, _guard_value, db_setting.value
2738 )
2739 )
2741 # A reconciled secret-bearing container is persisted
2742 # exactly as merged: coercing it through a str-typed
2743 # ui_element would flatten it to its Python repr and
2744 # disclose a just-restored secret as a plain string
2745 # (#6201). See _reconciled_container_value_for_write.
2746 if not _is_reconciled_container:
2747 value = coerce_setting_for_write(
2748 key=db_setting.key,
2749 value=value,
2750 ui_element=db_setting.ui_element,
2751 )
2753 # Validate against the setting's constraints (options
2754 # membership, numeric bounds, checkbox type) — the same
2755 # check save_all_settings (~line 655) and
2756 # api_update_setting (~line 3125) run. Main added this
2757 # exact call to the Flask route this function replaces
2758 # (fb49985aa8: "this JS-disabled POST fallback wrote
2759 # values unchecked"); the FastAPI rewrite dropped it.
2760 # Best-effort semantics preserved: an invalid key is
2761 # skipped and counted in failed_count, the rest of the
2762 # batch still validates, saves, and commits.
2763 #
2764 # A reconciled secret-bearing container skips
2765 # ``validate_setting`` the same way it skips coercion
2766 # above. The load-bearing reason is TYPE REJECTION, not
2767 # logging: ``validate_setting`` runs the value through
2768 # ``get_typed_setting_value`` for the row's DECLARED
2769 # ui_element, so a dict/list on a "number"/"range" row
2770 # comes back "Value must be a number" and the merged
2771 # edit -- with the secret just restored into it -- is
2772 # lost. Its other former reason (that same path logged
2773 # the raw, secret-bearing value at WARNING) is closed
2774 # at the converter itself as of #6201 round 6, so it no
2775 # longer justifies anything on its own;
2776 # ``_coerced_for_guard``'s comment already says this.
2777 # ACCEPTED LOSS: skipping validation also skips the
2778 # options-membership / min-max / checkbox-type checks,
2779 # so e.g. a "select" row that already holds a container
2780 # accepts a merged container with no options check.
2781 # Bounded to rows already holding a container under a
2782 # scalar ui_element -- a user-constructed shape -- and
2783 # the merge itself already resolved every leaf against
2784 # the stored row (#6201).
2785 if value is not None and not _is_reconciled_container:
2786 is_valid, error_message = validate_setting(
2787 db_setting, value
2788 )
2789 if not is_valid:
2790 logger.warning(
2791 f"Validation failed for setting {key}: "
2792 f"{error_message}"
2793 )
2794 failed_count += 1
2795 continue
2797 if not settings_manager.set_setting(key, value, commit=False):
2798 failed_count += 1
2799 logger.warning(
2800 "Failed to save setting {}", _log_value_preview(key)
2801 )
2802 else:
2803 changed_settings.append(key)
2804 except Exception:
2805 logger.exception(
2806 "Error saving setting {}", _log_value_preview(key)
2807 )
2808 failed_count += 1
2810 if rejected_count:
2811 logger.warning(
2812 f"Rejected {rejected_count} new setting(s) outside "
2813 "allowed namespaces"
2814 )
2816 try:
2817 db_session.commit()
2818 invalidate_settings_caches(username)
2819 reschedule_document_jobs_if_needed(username, changed_settings)
2820 reschedule_zotero_jobs_if_needed(username, changed_settings)
2821 except Exception:
2822 db_session.rollback()
2823 logger.exception("Failed to commit settings")
2824 return {
2825 "ok": False,
2826 "policy_error": None,
2827 "failed": failed_count + 1,
2828 "rejected": rejected_count,
2829 "sentinel_rejected": sentinel_rejected_ui_elements,
2830 "sentinel_rejected_too_deep": sentinel_rejected_too_deep,
2831 }
2833 return {
2834 "ok": failed_count == 0 and not sentinel_rejected_ui_elements,
2835 "policy_error": None,
2836 "failed": failed_count,
2837 "rejected": rejected_count,
2838 "sentinel_rejected": sentinel_rejected_ui_elements,
2839 "sentinel_rejected_too_deep": sentinel_rejected_too_deep,
2840 }
2843@router.post("/save_settings")
2844@settings_limit
2845async def save_settings(
2846 request: Request,
2847 username: Annotated[str, Depends(require_auth)],
2848):
2849 """Save all settings from the form using POST method — fallback when
2850 JavaScript is disabled. Body work offloaded to threadpool."""
2851 from ..dependencies.flash import flash
2853 outcome = None
2854 try:
2855 form_data = dict(await request.form())
2856 form_data.pop("csrf_token", None)
2857 outcome = await run_db_sync(_save_settings_sync, form_data, username)
2858 except Exception:
2859 logger.exception("Error in save_settings")
2861 # Give the no-JS user visible feedback via a flash message (rendered on
2862 # the /settings/ page) instead of a silent redirect. Flask flashed the
2863 # same success/error; the migration had dropped it.
2864 if outcome is None:
2865 flash(request, "Failed to save settings. Please try again.", "error")
2866 elif outcome.get("policy_error"): 2866 ↛ 2867line 2866 didn't jump to line 2867 because the condition on line 2866 was never true
2867 flash(request, outcome["policy_error"], "error")
2868 elif outcome.get("sentinel_rejected"):
2869 # Drop the empty-value hint if any rejected setting is a password
2870 # input, where an empty write is a no-op rather than a clear (see
2871 # ``_redaction_sentinel_error``).
2872 _rejected_uis = outcome["sentinel_rejected"]
2873 _too_deep = outcome.get("sentinel_rejected_too_deep") or []
2874 _hint_ui = (
2875 "password"
2876 if any(ui == "password" for ui in _rejected_uis)
2877 else None
2878 )
2879 # Every rejection in this batch was the depth cap, not an actual
2880 # sentinel -- say so instead of claiming the value "contains the
2881 # redaction placeholder" (see ``_container_too_deep_error``). A
2882 # MIXED batch (some too-deep, some an actual sentinel) states
2883 # both reasons instead of only the sentinel wording, which used
2884 # to silently drop the too-deep explanation for whichever
2885 # settings in the batch were rejected for that reason instead.
2886 if _too_deep and all(_too_deep): 2886 ↛ 2887line 2886 didn't jump to line 2887 because the condition on line 2886 was never true
2887 _message = _container_too_deep_error()
2888 elif _too_deep and any(_too_deep): 2888 ↛ 2889line 2888 didn't jump to line 2889 because the condition on line 2888 was never true
2889 _message = (
2890 _redaction_sentinel_error(_hint_ui)
2891 + " (Some of these were instead rejected for nesting too "
2892 "deeply to validate — flatten those before saving.)"
2893 )
2894 else:
2895 _message = _redaction_sentinel_error(_hint_ui)
2896 flash(
2897 request,
2898 f"Rejected {len(_rejected_uis)} settings: " + _message,
2899 "error",
2900 )
2901 elif outcome.get("failed"):
2902 flash(
2903 request,
2904 f"Saved with {outcome['failed']} setting(s) failing. "
2905 "Check the values and try again.",
2906 "warning",
2907 )
2908 elif outcome.get("rejected"):
2909 flash(
2910 request,
2911 f"Settings saved; {outcome['rejected']} unrecognized key(s) "
2912 "were ignored.",
2913 "warning",
2914 )
2915 else:
2916 flash(request, "Settings saved.", "success")
2918 return RedirectResponse(url="/settings/", status_code=302)
2921def _log_post_save_settings_error(error: SettingsValueError) -> None:
2922 """Log a settings error raised after a bulk save already committed.
2924 ``error.diagnostic`` holds only the key, its source and a fixed reason,
2925 never the value.
2926 """
2927 logger.error(
2928 "Settings saved, but a post-save step could not read settings: {}",
2929 error.diagnostic,
2930 )
2933def _settings_value_error_response(
2934 resource: str, error: SettingsValueError, *, bulk_save: bool = False
2935) -> JSONResponse:
2936 """Return the diagnostic for an invalid configured setting.
2938 The upgrade guide tells operators to locate the setting from this
2939 message, so it replaces the generic error. ``error.diagnostic`` holds
2940 only the key, its source and a fixed reason, never the value.
2941 Bulk saves retain their existing status/message response fields.
2942 """
2943 logger.error("Could not load {}: {}", resource, error.diagnostic)
2944 body = (
2945 {"status": "error", "message": error.diagnostic}
2946 if bulk_save
2947 else {"error": error.diagnostic}
2948 )
2949 return JSONResponse(body, status_code=500)
2952# API Routes
2953@router.get("/api")
2954def api_get_all_settings(
2955 request: Request,
2956 username: Annotated[str, Depends(require_auth)],
2957):
2958 """Get all settings"""
2959 try:
2960 # Get query parameters
2961 category = request.query_params.get("category")
2963 with get_user_db_session(username) as db_session:
2964 settings_manager = get_settings_manager(db_session, username)
2966 # Get settings
2967 settings = settings_manager.get_all_settings()
2969 # Filter by category if requested
2970 if category:
2971 filtered_settings = {}
2972 # Need to get all setting details to check category
2973 db_settings = db_session.query(Setting).all()
2974 category_keys = [
2975 s.key for s in db_settings if s.category == category
2976 ]
2978 # Filter settings by keys
2979 filtered_settings = {
2980 key: value
2981 for key, value in settings.items()
2982 if key in category_keys
2983 }
2985 settings = filtered_settings
2987 # Operator-gate the "unprotected" escape hatch out of the
2988 # egress-scope select's options unless explicitly enabled, and
2989 # normalise the displayed value (#5148 / 87537d9ec).
2990 if "policy.egress_scope" in settings:
2991 settings["policy.egress_scope"] = _shape_egress_scope_setting(
2992 "policy.egress_scope", settings["policy.egress_scope"]
2993 )
2995 # Hide the operator-gated unencrypted "filesystem" PDF-storage
2996 # option, same rationale as the egress-scope shaping above
2997 # (#5148 / fb49985aa).
2998 if "research_library.pdf_storage_mode" in settings:
2999 settings["research_library.pdf_storage_mode"] = (
3000 _shape_pdf_storage_mode_setting(
3001 "research_library.pdf_storage_mode",
3002 settings["research_library.pdf_storage_mode"],
3003 )
3004 )
3006 # Redact secret values (API keys, passwords, OAuth tokens) — main
3007 # does this (DataSanitizer.redact_settings_snapshot) as defense in
3008 # depth: this JSON gets cached by clients, logged by proxies, and
3009 # pasted into bug reports. Safe to round-trip: the save path treats
3010 # the '[REDACTED]' sentinel as a no-op (see the write-back guards in
3011 # _save_*_settings_sync), so the dashboard re-saving a redacted dump
3012 # never overwrites stored credentials, and settings.js seeds the
3013 # password baselines from this sentinel (PR #3947).
3014 from ...security.data_sanitizer import DataSanitizer
3016 settings = DataSanitizer.redact_settings_snapshot(settings)
3017 return {"status": "success", "settings": settings}
3018 except SettingsValueError as error:
3019 return _settings_value_error_response("settings", error)
3020 except Exception:
3021 logger.exception("Error getting settings")
3022 return JSONResponse(
3023 {"error": "Failed to retrieve settings"}, status_code=500
3024 )
3027@router.post("/api/import")
3028@settings_limit
3029def api_import_settings(
3030 request: Request,
3031 username: Annotated[str, Depends(require_auth)],
3032):
3033 """Import settings from defaults file"""
3034 try:
3035 with get_user_db_session(username) as db_session:
3036 settings_manager = get_settings_manager(db_session, username)
3038 # Settings lock (app.lock_settings) -- ported from main's
3039 # api_import_settings (#5659, "enforce the settings lock on delete,
3040 # import and reset"). SettingsManager already refuses when
3041 # locked, so the write cannot happen either way; this repeats
3042 # the check at the route so a locked instance answers 403
3043 # rather than 200 with nothing written, which is what main's
3044 # own comment gives as the reason. Without it the merge that
3045 # brought #5659 in would have silently dropped the fix, since
3046 # it landed in a Flask file this migration deletes.
3047 if settings_manager.settings_locked:
3048 return JSONResponse(
3049 {"error": "Settings are locked and cannot be imported"},
3050 status_code=403,
3051 )
3053 # See the identical call in reset_to_defaults: this flag is
3054 # required so a bulk import cannot overwrite env-locked settings,
3055 # and it defaults to False.
3056 settings_manager.load_from_defaults_file(
3057 preserve_environment_locked=True
3058 )
3059 imported_setting_keys = tuple(settings_manager.default_settings)
3061 invalidate_settings_caches(username)
3062 reschedule_document_jobs_if_needed(username, imported_setting_keys)
3063 reschedule_zotero_jobs_if_needed(username, imported_setting_keys)
3064 return {"message": "Settings imported successfully"}
3065 except Exception:
3066 logger.exception("Error importing settings")
3067 return JSONResponse(
3068 {"error": "Failed to import settings"}, status_code=500
3069 )
3072@router.get("/api/categories")
3073def api_get_categories(
3074 request: Request,
3075 username: Annotated[str, Depends(require_auth)],
3076):
3077 """Get all setting categories"""
3078 try:
3079 with get_user_db_session(username) as db_session:
3080 # Get all distinct categories
3081 categories = db_session.query(Setting.category).distinct().all()
3082 category_list = [c[0] for c in categories if c[0] is not None]
3084 return {"categories": category_list}
3085 except Exception:
3086 logger.exception("Error getting categories")
3087 return JSONResponse(
3088 {"error": "Failed to retrieve settings"}, status_code=500
3089 )
3092@router.get("/api/types")
3093def api_get_types(
3094 request: Request, username: Annotated[str, Depends(require_auth)]
3095):
3096 """Get all setting types"""
3097 try:
3098 # Get all setting types
3099 types = [t.value for t in SettingType]
3100 return {"types": types}
3101 except Exception:
3102 logger.exception("Error getting types")
3103 return JSONResponse(
3104 {"error": "Failed to retrieve settings"}, status_code=500
3105 )
3108@router.get("/api/ui_elements")
3109def api_get_ui_elements(
3110 request: Request, username: Annotated[str, Depends(require_auth)]
3111):
3112 """Get all UI element types"""
3113 try:
3114 # The types a settings read can convert. A new setting with any
3115 # other ui_element is rejected on write.
3116 ui_elements = list(UI_ELEMENT_TO_SETTING_TYPE)
3118 return {"ui_elements": ui_elements}
3119 except Exception:
3120 logger.exception("Error getting UI elements")
3121 return JSONResponse(
3122 {"error": "Failed to retrieve settings"}, status_code=500
3123 )
3126@router.get("/api/available-models")
3127@router.post("/api/available-models")
3128async def api_get_available_models(
3129 request: Request, username: Annotated[str, Depends(require_auth)]
3130):
3131 """Read cached models on GET; discover or refresh only on POST.
3133 ``GET`` returns only the policy-filtered model cache: a cold, expired
3134 or filtered cache answers with an empty model list, and a ``GET`` never
3135 performs provider-specific credential lookups, contacts a provider or
3136 writes the cache. It still loads the strict settings snapshot for policy.
3137 Discovery calls providers with the user's stored API keys and rewrites
3138 their cached ``ProviderModel`` rows, so it is a state change accepted
3139 only on the CSRF-protected ``POST``:
3141 * a ``POST`` without a refresh query returns the fresh cached rows and
3142 discovers every provider only on a cache miss;
3143 * ``POST ?force_refresh=true`` bypasses and replaces the whole cache;
3144 * ``POST ?refresh_provider=<p>`` re-discovers that provider (or every
3145 provider, when the cache has no usable rows). A target takes
3146 precedence over ``force_refresh=true``, so a targeted refresh is never
3147 widened into a full one.
3149 A ``GET`` carrying ``refresh_provider`` or ``force_refresh=true``
3150 answers 405 (``Allow: POST``) without touching the cache or any
3151 provider.
3153 Discovery runs on its own executor and does not continue the provider
3154 loop after cancellation.
3155 """
3156 # Parse as bool — a raw string (non-empty) is always truthy, so the old
3157 # code always took the force-refresh path, deleting every cached model
3158 # on every request.
3159 force_refresh_requested = (
3160 request.query_params.get("force_refresh", "false").lower() == "true"
3161 )
3162 # Refused here, on the event loop, so a 405 never waits for a worker.
3163 if request.method != "POST" and (
3164 force_refresh_requested or request.query_params.get("refresh_provider")
3165 ):
3166 raise HTTPException(
3167 status_code=405,
3168 detail="Use POST to refresh models",
3169 headers={"Allow": "POST"},
3170 )
3171 cancelled = threading.Event()
3172 try:
3173 # Cache-only reads must not queue behind slow provider discovery.
3174 if request.method != "POST":
3175 return await run_db_sync(
3176 _get_available_models, request, username, cancelled
3177 )
3178 # Discovery can wait out one provider timeout after another. Keep
3179 # those calls off the workers used by authenticated DB requests.
3180 return await run_model_discovery_sync(
3181 _get_available_models, request, username, cancelled
3182 )
3183 finally:
3184 # Canceling an awaiting task cannot interrupt synchronous SDK I/O.
3185 # The worker must stop after its current provider returns, before
3186 # starting another request or replacing the user's cached models.
3187 cancelled.set()
3190def _get_available_models(
3191 request: Request, username: str, cancelled: threading.Event
3192):
3193 """Run discovery and DB access on the worker that owns their cleanup."""
3195 def check_cancelled():
3196 if cancelled.is_set():
3197 raise asyncio.CancelledError
3199 check_cancelled()
3200 # Only the CSRF-protected POST may discover providers or write the model
3201 # cache. Any other method reads the policy-filtered cache and stops there.
3202 discovery_allowed = request.method == "POST"
3203 endpoint_start = time.perf_counter()
3204 try:
3205 # request comes from FastAPI parameter
3206 from ...database.models.providers import ProviderModel
3208 # Optional targeted refresh: re-discover ONLY this provider and
3209 # merge the result into the cached catalog (used after a provider
3210 # API key is saved — the plain cache read would keep that
3211 # provider's stale/empty list for up to 24h, and a full refresh
3212 # re-probes every provider serially). POST-only: a GET carrying it
3213 # was refused with 405 above.
3214 refresh_provider = (
3215 normalize_provider(
3216 request.query_params.get("refresh_provider", "") or ""
3217 )
3218 if discovery_allowed
3219 else None
3220 )
3222 # The forced (full) refresh: POST ?force_refresh=true without a
3223 # target. A POST without the flag stays cache-first (it discovers
3224 # only on a cache miss), and a target takes precedence over the
3225 # flag, so a targeted POST stays targeted.
3226 force_refresh = (
3227 discovery_allowed
3228 and not refresh_provider
3229 and request.query_params.get("force_refresh", "false").lower()
3230 == "true"
3231 )
3233 # Resolve the egress policy BEFORE the model cache is read or any
3234 # provider is discovered/contacted. Fails closed (PolicyDeniedError).
3235 policy_context, policy_snapshot = _resolve_model_discovery_policy(
3236 username
3237 )
3239 # Get all auto-discovered providers. Every provider is advertised to
3240 # the UI; the egress policy marks blocked ones as disabled (with a
3241 # reason) instead of removing them. Removing them silently made the
3242 # "Model Provider" dropdown look short after configuring a cloud API
3243 # key, leading users to assume the key wasn't picked up — and left
3244 # the front-end code that renders the reason
3245 # (static/js/components/custom_dropdown.js, research.js, settings.js)
3246 # unreachable. The per-provider MODELS list below is still filtered
3247 # so we never actually call a blocked provider (#5922 / #5662).
3248 from ...llm.providers import get_discovered_provider_options
3250 options = list(get_discovered_provider_options())
3251 provider_options = _provider_options_for_snapshot(
3252 options, policy_context, policy_snapshot
3253 )
3254 # Grading uses its own endpoint override. Keep this copied snapshot
3255 # out of research model discovery, credential reads and model caches.
3256 #
3257 # The override is read back from the SAME snapshot the policy context
3258 # was built from, not through a second settings read: the snapshot
3259 # already carries the LDR_* overlay for this key, so the two agree
3260 # (no TOCTOU between classifying the endpoint and reading it), and it
3261 # costs one fewer get_user_db_session per request. The route's own
3262 # deny-before-credential-read ordering is unaffected either way:
3263 # _resolve_model_discovery_policy above has already opened a session
3264 # and read the whole snapshot before anything here runs, and the
3265 # per-provider api_key_setting reads still happen only after the
3266 # denial point.
3267 benchmark_snapshot = dict(policy_snapshot)
3268 benchmark_endpoint = unwrap_setting(
3269 policy_snapshot.get(
3270 "benchmark.evaluation.endpoint_url",
3271 "https://openrouter.ai/api/v1",
3272 )
3273 )
3274 # Match get_llm's explicit endpoint-override normalization.
3275 benchmark_snapshot["llm.openai_endpoint.url"] = (
3276 benchmark_endpoint.strip()
3277 if isinstance(benchmark_endpoint, str)
3278 else benchmark_endpoint
3279 )
3280 benchmark_provider_options = _provider_options_for_snapshot(
3281 options, policy_context, benchmark_snapshot
3282 )
3284 # Add remaining hardcoded providers (complex local providers not yet migrated)
3286 # Available models by provider
3287 providers: dict[str, Any] = {}
3289 # Cached rows of a targeted refresh's provider, set aside when the
3290 # target's cache entries are dropped for re-fetch. Restored to the
3291 # response if the re-fetch fails, so a transient upstream error
3292 # can't blank the provider's catalog.
3293 stale_target_rows: list | None = None
3294 # Whether the cache read produced ANY rows (before the targeted
3295 # refresh pops its provider). Distinct from a post-pop empty
3296 # ``providers`` dict, which just means the cache held only the
3297 # target — not that it is cold.
3298 cache_had_rows = False
3300 # Check database cache first (unless force_refresh is True)
3301 if not force_refresh:
3302 try:
3303 # Define cache expiration (24 hours)
3304 cache_expiry = datetime.now(UTC) - timedelta(hours=24)
3306 # Get cached models from database
3307 with get_user_db_session(username) as db_session:
3308 cached_models = (
3309 db_session.query(ProviderModel)
3310 .filter(ProviderModel.last_updated > cache_expiry)
3311 .all()
3312 )
3314 if cached_models:
3315 logger.info(
3316 f"Found {len(cached_models)} cached models in database"
3317 )
3319 # Group models by provider
3320 for model in cached_models:
3321 if (
3322 policy_context.require_local_llm
3323 and not _model_discovery_provider_allowed(
3324 model.provider, policy_context, policy_snapshot
3325 )
3326 ):
3327 continue
3328 provider_key = (
3329 f"{normalize_provider(model.provider)}_models"
3330 )
3331 if provider_key not in providers: 3331 ↛ 3334line 3331 didn't jump to line 3334 because the condition on line 3331 was always true
3332 providers[provider_key] = []
3334 providers[provider_key].append(
3335 {
3336 "value": model.model_key,
3337 "label": model.model_label,
3338 "provider": model.provider.upper(),
3339 }
3340 )
3342 # If we have cached data for all providers, return it —
3343 # unless a targeted refresh was requested for a provider
3344 # whose rows are stale or absent (e.g. right after its
3345 # API key was saved): drop just that provider's cached
3346 # entries and fall through to a single-provider fetch.
3347 cache_had_rows = bool(providers)
3348 if providers and refresh_provider:
3349 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: Dictionary key only
3350 stale_key = f"{refresh_provider}_models"
3351 # Drop the target's cached entries (if any) and
3352 # fall through to a single-provider re-fetch. Keep
3353 # them aside for restoration if the re-fetch fails.
3354 stale_target_rows = providers.pop(stale_key, None)
3355 elif providers:
3356 _log_available_models_duration(
3357 endpoint_start, cache_hit=True
3358 )
3359 logger.info("Returning cached models from database")
3360 return {
3361 "provider_options": provider_options,
3362 "benchmark_provider_options": benchmark_provider_options,
3363 "providers": providers,
3364 }
3366 except PolicyDeniedError:
3367 raise
3368 except Exception:
3369 logger.warning("Error reading cached models from database")
3370 if not discovery_allowed: 3370 ↛ 3377line 3370 didn't jump to line 3377 because the condition on line 3370 was always true
3371 raise
3372 # A CSRF-protected POST may recover by fetching fresh data.
3374 # Even an ordinary cache miss can populate or replace persisted rows.
3375 # GET must never reach model listing, credential lookup or cache writes.
3376 # Return the same response shape for cold, expired or filtered caches.
3377 if not discovery_allowed:
3378 _log_available_models_duration(endpoint_start, cache_hit=False)
3379 return {
3380 "provider_options": provider_options,
3381 "benchmark_provider_options": benchmark_provider_options,
3382 "providers": providers,
3383 }
3385 if refresh_provider and not cache_had_rows:
3386 # Cold cache (fresh install, or every row older than 24h): a
3387 # targeted refresh would return a one-provider catalog that
3388 # the client would cache as the whole world. Fall back to a
3389 # full discovery — the plain cold-cache path does exactly that.
3390 logger.info(
3391 "Targeted refresh with an empty model cache: "
3392 "falling back to full discovery"
3393 )
3394 refresh_provider = None
3396 # Ollama / OpenAI / Anthropic model listing is handled by the
3397 # auto-discovery loop below (their provider classes' list_models_for_api),
3398 # leaving one provider-class fetch path. Those classes own base-URL
3399 # validation and authentication, and the router does no duplicate network
3400 # work. The removed Ollama probe used safe_get's request-time SSRF checks;
3401 # its problem here was duplicate fetching, not missing validation.
3403 # Fetch models from auto-discovered providers
3404 from ...llm.providers import discover_providers
3406 discovered_providers = discover_providers()
3408 # If the effective egress posture is local-only, the user (or policy)
3409 # has opted into local-only inference, so don't list cloud providers
3410 # (OpenRouter, Google, XAI, IonOS, OpenAI, Anthropic, ...). We still
3411 # list the local providers via their provider classes — which validate
3412 # the URL (SSRF) and support auth headers — by filtering the discovered
3413 # set rather than skipping discovery entirely. The filter classifies
3414 # each provider's configured ENDPOINT (evaluate_llm_endpoint), so a
3415 # nominally-local provider pointed at a public URL is dropped too; a
3416 # static LOCAL_PROVIDERS name check would have kept it.
3417 if policy_context.require_local_llm:
3418 all_discovered_providers = discovered_providers
3419 discovered_providers = {
3420 key: info
3421 for key, info in all_discovered_providers.items()
3422 if _model_discovery_provider_allowed(
3423 key, policy_context, policy_snapshot
3424 )
3425 }
3426 logger.bind(policy_audit=True).info(
3427 "local-only egress posture: limiting discovered model lists "
3428 "to endpoint-approved local providers",
3429 kept=list(discovered_providers.keys()),
3430 skipped=[
3431 key
3432 for key in all_discovered_providers
3433 if key not in discovered_providers
3434 ],
3435 )
3437 if refresh_provider and not force_refresh:
3438 # Targeted refresh: only the named provider is re-fetched (and
3439 # only if it survived the policy filter above). Unknown
3440 # provider names degrade to a plain cache read (nothing to
3441 # re-fetch) — the merged `providers` dict below still returns
3442 # the other providers' cached rows.
3443 discovered_providers = {
3444 key: info
3445 for key, info in discovered_providers.items()
3446 if key.lower() == refresh_provider
3447 }
3449 targeted_fetch_failed = False
3450 fetched_provider_settings = {}
3451 for provider_key, provider_info in discovered_providers.items():
3452 check_cancelled()
3453 provider_models = []
3454 try:
3455 logger.info(
3456 f"Fetching models from {provider_info.provider_name}"
3457 )
3459 # Get the provider class
3460 provider_class = provider_info.provider_class
3462 # Get API key if configured
3463 api_key = _get_setting_from_session(
3464 provider_class.api_key_setting, username, ""
3465 )
3467 # Get base URL if provider has configurable URL
3468 provider_base_url: str | None = None
3469 url_setting = getattr(provider_class, "url_setting", None)
3470 if url_setting:
3471 provider_base_url = _get_setting_from_session(
3472 url_setting, username, ""
3473 )
3475 fetched_settings = {}
3476 if provider_class.api_key_setting is not None:
3477 fetched_settings[provider_class.api_key_setting] = api_key
3478 if url_setting:
3479 fetched_settings[url_setting] = provider_base_url
3480 fetched_provider_settings[
3481 f"{normalize_provider(provider_key)}_models"
3482 ] = fetched_settings
3484 if policy_context.require_local_llm:
3485 provider_snapshot = dict(policy_snapshot)
3486 if url_setting:
3487 provider_snapshot[url_setting] = provider_base_url
3488 if not _model_discovery_provider_allowed( 3488 ↛ 3491line 3488 didn't jump to line 3491 because the condition on line 3488 was never true
3489 provider_key, policy_context, provider_snapshot
3490 ):
3491 continue
3493 # Use the provider's list_models_for_api method. This is the
3494 # single remaining cloud fetch path (OpenAI's models.list()
3495 # among them), so it carries PR #3483's provider-fetch timer.
3496 provider_fetch_start = time.perf_counter()
3497 models = provider_class.list_models_for_api(
3498 api_key, provider_base_url
3499 )
3500 provider_fetch_ms = (
3501 time.perf_counter() - provider_fetch_start
3502 ) * 1000
3503 if provider_fetch_ms > 1000:
3504 logger.info(
3505 f"{provider_key} list_models_for_api took "
3506 f"{provider_fetch_ms:.0f}ms"
3507 )
3508 else:
3509 logger.debug(
3510 f"{provider_key} list_models_for_api took "
3511 f"{provider_fetch_ms:.0f}ms"
3512 )
3514 # Format models for the API response
3515 for model in models:
3516 provider_models.append(
3517 {
3518 "value": model["value"],
3519 "label": model[
3520 "label"
3521 ], # Use provider's label as-is
3522 "provider": provider_key,
3523 }
3524 )
3526 logger.info(
3527 f"Successfully fetched {len(provider_models)} models from {provider_info.provider_name}"
3528 )
3530 except PolicyDeniedError:
3531 raise
3532 except Exception:
3533 logger.exception(
3534 f"Error getting {provider_info.provider_name} models"
3535 )
3536 if (
3537 refresh_provider
3538 and provider_key.lower() == refresh_provider
3539 ):
3540 # The targeted provider's re-fetch failed: don't let a
3541 # transient upstream error blank its catalog — the
3542 # cached rows set aside earlier are restored below and
3543 # the DB rows are left untouched.
3544 targeted_fetch_failed = True
3546 # Set models in providers dict using lowercase key
3547 providers[f"{normalize_provider(provider_key)}_models"] = (
3548 provider_models
3549 )
3550 logger.info(
3551 f"Final {provider_key} models count: {len(provider_models)}"
3552 )
3554 # Also check after the final provider, including failed fetches.
3555 check_cancelled()
3557 # Restore the targeted provider's cached rows when its re-fetch
3558 # failed (response keeps the last-known-good catalog).
3559 if targeted_fetch_failed and stale_target_rows is not None:
3560 providers[f"{refresh_provider}_models"] = stale_target_rows
3562 # ``list_models_for_api`` converts every upstream failure into an
3563 # empty list (its catch-all never raises), so the exception path
3564 # above cannot fire for real network errors. Treat "targeted
3565 # refresh produced zero models while non-empty cached rows were
3566 # set aside" as a failed probe too: keep the last-known-good
3567 # catalog and skip the cache write, so a transient outage during
3568 # a key re-save cannot blank the provider's cached models.
3569 if (
3570 refresh_provider
3571 and not force_refresh
3572 and stale_target_rows
3573 and not providers.get(f"{refresh_provider}_models")
3574 ):
3575 targeted_fetch_failed = True
3576 providers[f"{refresh_provider}_models"] = stale_target_rows
3578 # Save fetched models to database cache. A targeted refresh only
3579 # persists the refreshed provider's rows — other providers keep
3580 # their cached rows and TTLs untouched. A failed targeted re-fetch
3581 # persists nothing (the cached rows stay valid).
3582 providers_to_save = providers
3583 if refresh_provider and not force_refresh:
3584 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: Dictionary key only
3585 target_key = f"{refresh_provider}_models"
3586 providers_to_save = (
3587 {}
3588 if targeted_fetch_failed
3589 else {
3590 key: models
3591 for key, models in providers.items()
3592 if key == target_key
3593 }
3594 )
3595 if force_refresh or providers_to_save:
3596 # We fetched fresh data, save it to database
3597 with get_user_db_session(username) as db_session:
3598 try:
3599 # Take SQLite's writer lock only AFTER all network calls.
3600 # Checking settings and writing models under the same lock
3601 # prevents a key/URL change (including in another worker)
3602 # from racing between the check and the cache replacement.
3603 db_session.connection().exec_driver_sql("BEGIN IMMEDIATE")
3604 current_providers_to_save = {
3605 key: models
3606 for key, models in providers_to_save.items()
3607 if _model_discovery_settings_unchanged(
3608 db_session, fetched_provider_settings.get(key, {})
3609 )
3610 }
3611 superseded = len(current_providers_to_save) != len(
3612 providers_to_save
3613 )
3614 providers_to_save = current_providers_to_save
3615 # A superseded full refresh must also leave the newer
3616 # provider's rows intact instead of clearing the catalog.
3617 if force_refresh and not superseded:
3618 # When force refresh, clear ALL cached models to remove any stale data
3619 # from old code versions or deleted providers
3620 deleted_count = db_session.query(ProviderModel).delete()
3621 logger.info(
3622 f"Force refresh: cleared all {deleted_count} cached models"
3623 )
3624 else:
3625 # Clear old cache entries only for providers we're updating
3626 for provider_key in providers_to_save:
3627 provider_name = provider_key.replace(
3628 "_models", ""
3629 ).upper()
3630 db_session.query(ProviderModel).filter(
3631 ProviderModel.provider == provider_name
3632 ).delete()
3634 # Insert new models. A targeted refresh stamps its
3635 # rows with the oldest ``last_updated`` among the other
3636 # providers' fresh cached rows so the whole catalog
3637 # still expires at once: the cache read returns any
3638 # fresh rows as the complete catalog, so rows that
3639 # outlived the others would hide every other
3640 # provider's models until they expired too. Read under
3641 # the writer lock, so a full refresh that committed
3642 # after this request's cache read is taken into
3643 # account.
3644 row_time = datetime.now(UTC)
3645 if refresh_provider and not force_refresh:
3646 other_row_times = [
3647 model.last_updated
3648 for model in db_session.query(ProviderModel)
3649 .filter(
3650 ProviderModel.last_updated
3651 > row_time - timedelta(hours=24)
3652 )
3653 .all()
3654 if normalize_provider(model.provider)
3655 != refresh_provider
3656 and isinstance(model.last_updated, datetime)
3657 ]
3658 if other_row_times:
3659 row_time = min(
3660 row_time,
3661 *(
3662 t if t.tzinfo else t.replace(tzinfo=UTC)
3663 for t in other_row_times
3664 ),
3665 )
3666 for provider_key, models in providers_to_save.items():
3667 provider_name = provider_key.replace(
3668 "_models", ""
3669 ).upper()
3670 for model in models:
3671 if ( 3671 ↛ 3670line 3671 didn't jump to line 3670 because the condition on line 3671 was always true
3672 isinstance(model, dict)
3673 and "value" in model
3674 and "label" in model
3675 ):
3676 new_model = ProviderModel(
3677 provider=provider_name,
3678 model_key=model["value"],
3679 model_label=model["label"],
3680 last_updated=row_time,
3681 )
3682 db_session.add(new_model)
3684 db_session.commit()
3685 logger.info("Successfully cached models to database")
3687 except Exception:
3688 logger.exception("Error saving models to database cache")
3689 db_session.rollback()
3691 # Return all options
3692 _log_available_models_duration(endpoint_start, cache_hit=False)
3693 return {
3694 "provider_options": provider_options,
3695 "benchmark_provider_options": benchmark_provider_options,
3696 "providers": providers,
3697 }
3699 except PolicyDeniedError as exc:
3700 reason = exc.decision.reason
3701 status_code = 503 if reason == "settings_unavailable" else 400
3702 logger.bind(policy_audit=True).warning(
3703 "available-model discovery denied by egress policy", reason=reason
3704 )
3705 _log_available_models_duration(
3706 endpoint_start, cache_hit=False, error=True
3707 )
3708 return JSONResponse(
3709 {
3710 "status": "error",
3711 "message": f"Egress policy refused this request: {reason}",
3712 },
3713 status_code=status_code,
3714 )
3715 except Exception:
3716 logger.exception("Error getting available models")
3717 _log_available_models_duration(
3718 endpoint_start, cache_hit=False, error=True
3719 )
3720 return JSONResponse(
3721 {
3722 "status": "error",
3723 "message": "Failed to retrieve available models",
3724 },
3725 status_code=500,
3726 )
3729def _log_available_models_duration(
3730 start: float, cache_hit: bool, error: bool = False
3731) -> None:
3732 """Log /api/available-models endpoint duration.
3734 Uses INFO when the endpoint took > 1s (indicating a real provider fetch
3735 latency worth flagging), DEBUG otherwise. This is the likely culprit for
3736 Path C (LLM provider timeout masquerading as backend hang) in the
3737 login-hang investigation (PR #3483 / #5961).
3738 """
3739 elapsed_ms = (time.perf_counter() - start) * 1000
3740 path = (
3741 "error"
3742 if error
3743 else ("cache hit" if cache_hit else "full provider fetch")
3744 )
3745 if elapsed_ms > 1000:
3746 logger.info(f"/api/available-models ({path}) took {elapsed_ms:.0f}ms")
3747 else:
3748 logger.debug(f"/api/available-models ({path}) took {elapsed_ms:.0f}ms")
3751def _get_engine_icon_and_category(
3752 engine_data: dict, engine_class=None
3753) -> tuple:
3754 """
3755 Get icon emoji and category label for a search engine based on its attributes.
3757 Args:
3758 engine_data: Engine configuration dictionary
3759 engine_class: Optional loaded engine class to check attributes
3761 Returns:
3762 Tuple of (icon, category) strings
3763 """
3764 # Check attributes from either the class or the engine data
3765 if engine_class:
3766 is_scientific = getattr(engine_class, "is_scientific", False)
3767 is_generic = getattr(engine_class, "is_generic", False)
3768 is_local = getattr(engine_class, "is_local", False)
3769 is_news = getattr(engine_class, "is_news", False)
3770 is_code = getattr(engine_class, "is_code", False)
3771 else:
3772 is_scientific = engine_data.get("is_scientific", False)
3773 is_generic = engine_data.get("is_generic", False)
3774 is_local = engine_data.get("is_local", False)
3775 is_news = engine_data.get("is_news", False)
3776 is_code = engine_data.get("is_code", False)
3778 # Check books attribute
3779 if engine_class:
3780 is_books = getattr(engine_class, "is_books", False)
3781 else:
3782 is_books = engine_data.get("is_books", False)
3784 # Return icon and category based on engine type
3785 # Priority: local > scientific > news > code > books > generic > default
3786 if is_local:
3787 return "📁", "Local RAG"
3788 if is_scientific:
3789 return "🔬", "Scientific"
3790 if is_news:
3791 return "📰", "News"
3792 if is_code:
3793 return "💻", "Code"
3794 if is_books:
3795 return "📚", "Books"
3796 if is_generic:
3797 return "🌐", "Web Search"
3798 return "🔍", "Search"
3801# Ported verbatim from the Flask ``settings_routes`` (#5221 / issue
3802# #5204). The body is framework-agnostic — it only consults the egress
3803# PDP and the logger — so nothing needed adapting; only its caller did.
3804# This landed on main against a module this branch had already deleted,
3805# so the merge dropped it while keeping the frontend that depends on it.
3806def _classify_options_for_egress(
3807 engine_options: list,
3808 *,
3809 egress_scope: str,
3810 primary_engine: str,
3811 settings_snapshot: dict,
3812 username: Optional[str],
3813 search_engines: Optional[dict] = None,
3814) -> None:
3815 """Stamp each engine option with an ``egress: {allowed, reason}`` field.
3817 Used by ``api_get_available_search_engines`` when the caller passes
3818 ``?egress_scope=…&primary=…``: the frontend needs to know which
3819 options are selectable under the active scope so it can disable
3820 (or hide) the rest in the dropdown. Issue #5204.
3822 The decision is taken through the same PDP the request-boundary
3823 precheck uses (``evaluate_engine``) so the dropdown's disabled set
3824 and the server's 400 denials stay perfectly aligned. Dropdown filtering
3825 strictly reflects PDP allowed decisions; frontend selection reconciliation
3826 handles updating any invalid primary selection when scope changes. The factory PEP
3827 still enforces at instantiation time; this is a UX filter, not a security boundary.
3829 After the PDP pass it applies ``_apply_guarded_url_overlay``, which
3830 can deny an engine the PDP allowed (an unapproved private instance
3831 URL). That overlay is scope-independent and therefore also runs when
3832 the caller did NOT opt into scope filtering — see the route — and on
3833 every degraded early return below (an unimportable policy module, an
3834 unknown ``?egress_scope=``): it needs only the settings snapshot, not
3835 the PDP context, and the runtime blocks the URL regardless, so
3836 skipping it there would make a guard-denied engine read as selectable.
3838 Operates in place; safe to call on a freshly built option list.
3839 Failures are swallowed per option (logged) — a single bad engine
3840 must not blank the whole dropdown.
3841 """
3842 try:
3843 from ...security.egress.policy import (
3844 context_from_snapshot,
3845 evaluate_engine,
3846 resolve_run_primary_engine,
3847 )
3848 except Exception: # pragma: no cover - defensive
3849 logger.exception(
3850 "egress policy unavailable; emitting unfiltered options"
3851 )
3852 # The URL guard does not ride on the PDP — it only needs the
3853 # snapshot — so it still runs when the PDP is unavailable.
3854 _apply_guarded_url_overlay(engine_options, settings_snapshot)
3855 return
3857 if not isinstance(settings_snapshot, dict):
3858 # Programming error: the route always passes a snapshot. Log it
3859 # loudly so a future caller-side change is debuggable instead of
3860 # silently falling back to the unfiltered list (PR #5221 review).
3861 logger.error(
3862 "_classify_options_for_egress received a non-dict "
3863 "settings_snapshot (type={}); emitting unfiltered options",
3864 type(settings_snapshot).__name__,
3865 )
3866 # Match every other degraded early return: the overlay is a
3867 # no-op here (it guards on isinstance(settings_snapshot, dict)
3868 # itself) since there's no dict to read a URL from, but calling
3869 # it keeps this branch consistent with the docstring above and
3870 # fail-safe if a future caller ever reaches this branch with
3871 # something the overlay CAN read.
3872 _apply_guarded_url_overlay(engine_options, settings_snapshot)
3873 return
3875 # Build an evaluation snapshot that reflects the requested scope and
3876 # primary engine parameter overrides sent by the frontend (issue #5204).
3877 eval_snapshot = dict(settings_snapshot)
3878 if egress_scope:
3879 eval_snapshot["policy.egress_scope"] = egress_scope
3880 if primary_engine:
3881 eval_snapshot["search.tool"] = primary_engine
3883 # The precheck builds an EgressContext via context_from_snapshot; the
3884 # adaptive resolution requires a primary, so fall back to the supplied
3885 # primary and ultimately the saved search.tool.
3886 try:
3887 try:
3888 primary = primary_engine or resolve_run_primary_engine(
3889 eval_snapshot
3890 )
3891 except ValueError:
3892 # ``resolve_run_primary_engine`` raises ValueError when
3893 # ``search.tool`` is missing/blank/non-string and no
3894 # ``default`` is given. That's the only documented failure
3895 # mode — narrow the catch so unrelated bugs (e.g. an
3896 # AttributeError on a future code path) surface instead of
3897 # being silently swallowed (PR #5221 review).
3898 primary = primary_engine or eval_snapshot.get("search.tool", "")
3899 try:
3900 ctx = context_from_snapshot(
3901 eval_snapshot,
3902 primary,
3903 username=username,
3904 )
3905 except Exception:
3906 # ``context_from_snapshot`` raises ``PolicyDeniedError`` for
3907 # an unknown scope (``unknown_egress_scope``) and ``ValueError``
3908 # for a malformed snapshot; either way the precheck would
3909 # 400 the run. Stamp a permissive decision so the frontend
3910 # falls back to the precheck instead of a half-blanked
3911 # dropdown. Use logger.exception (dev-only traceback, no
3912 # exc_info kwarg on logger.warning which would expose a
3913 # full traceback at WARNING level in production — PR #5221
3914 # review).
3915 logger.exception(
3916 "_classify_options_for_egress: context build failed; "
3917 "emitting permissive policy_unavailable decisions"
3918 )
3919 for opt in engine_options:
3920 opt["egress"] = {
3921 "allowed": True,
3922 "reason": "policy_unavailable",
3923 }
3924 # Permissive on SCOPE only. The URL guard is independent of
3925 # the PDP context that just failed to build, and the runtime
3926 # still blocks the URL, so a guard-denied engine must not read
3927 # as selectable on this degraded path either.
3928 _apply_guarded_url_overlay(engine_options, eval_snapshot)
3929 return
3930 except Exception: # pragma: no cover - defensive
3931 logger.exception(
3932 "egress context build failed; emitting unfiltered options"
3933 )
3934 _apply_guarded_url_overlay(engine_options, eval_snapshot)
3935 return
3937 for opt in engine_options:
3938 engine_id = opt.get("value")
3939 try:
3940 metadata = search_engines.get(engine_id) if search_engines else None
3941 decision = evaluate_engine(
3942 engine_id,
3943 ctx,
3944 settings_snapshot=eval_snapshot,
3945 metadata=metadata,
3946 )
3947 opt["egress"] = {
3948 "allowed": bool(decision.allowed),
3949 "reason": decision.reason,
3950 }
3951 except Exception: # pragma: no cover - defensive
3952 logger.exception(f"egress decision failed for engine {engine_id}")
3953 opt["egress"] = {"allowed": True, "reason": "decision_error"}
3955 # Guarded engine-URL overlay: a public engine whose configured private
3956 # URL lacks operator approval self-disables at runtime, so the dropdown
3957 # should gray it out too. Applied here (rather than in the caller) so
3958 # every code path that stamps egress decisions also carries the guard.
3959 # Deliberately scope-INDEPENDENT — see _apply_guarded_url_overlay.
3960 _apply_guarded_url_overlay(engine_options, eval_snapshot)
3963def _apply_guarded_url_overlay(
3964 engine_options: list, settings_snapshot: dict
3965) -> None:
3966 """Gray out engines whose configured instance URL the runtime will refuse.
3968 A PUBLIC-nature engine (SearXNG and friends) pointed at a private /
3969 loopback instance URL without operator approval self-disables at init
3970 and silently contributes zero results to the run. This overlay stamps
3971 ``egress={"allowed": False, "reason": "private_url_unapproved"}`` on
3972 those options so the picker grays them out with an explanatory note.
3974 SCOPE-INDEPENDENT BY CONSTRUCTION. The approval resolver this rides on
3975 (``resolve_engine_allow_private_ips``) deliberately ignores
3976 ``policy.egress_scope`` — it is a user-editable dropdown, so honouring
3977 it would be a self-service SSRF bypass — which means the runtime blocks
3978 the URL under EVERY scope. The overlay must therefore run under every
3979 scope too, including the shipped ``adaptive`` default and the
3980 ``unprotected`` escape hatch, and NOT only when the caller opted into
3981 scope filtering via ``?egress_scope=``. Before PR #5648's follow-up
3982 this lived inside the ``apply_egress_filter`` branch and could only
3983 ever fire under ``public_only`` (under ``private_only`` a public engine
3984 is already denied for scope reasons), i.e. never on a default install.
3986 ADVISORY, NOT AUTHORITATIVE — it is a strict subset of what the runtime
3987 enforces. ``check_private_engine_url_blocked`` is deliberately
3988 DNS-free (it must not turn a settings-page render into a resolver
3989 round-trip, nor let a hostile URL make the render path probe an
3990 internal name), so a private-DNS hostname (``searx.internal`` -> 10.x)
3991 and a short-form literal (``127.1``) are NOT caught here even though
3992 the runtime's ``validate_url`` resolves and blocks them. The overlay
3993 therefore fails OPEN: an engine it leaves selectable may still be
3994 refused at run time. For that residual gap the ONLY backstop is the
3995 runtime's log-panel ERROR — NOT the research form's warning banner,
3996 which ``warning_checks`` emits from this very same DNS-free check and
3997 which therefore stays silent on exactly the URLs the overlay misses
3998 (and is additionally suppressed once the user dismisses it, which the
3999 overlay ignores).
4001 It effectively never fails CLOSED — nothing the runtime would fetch is
4002 grayed out here. The one theoretical exception is a host name ending
4003 in ``.local``, which ``is_private_ip`` classifies by suffix: if a
4004 non-conforming resolver or a ``hosts`` entry maps it to a PUBLIC
4005 address the runtime would allow it while this overlay grays it out.
4006 ``.local`` is reserved for mDNS (RFC 6762), so that is a
4007 misconfiguration rather than a supported setup.
4009 Options already carrying a denial (``allowed: False``) are left alone
4010 so a scope denial keeps its more specific reason. Operates in place;
4011 every failure is swallowed per engine so one bad descriptor cannot
4012 blank the dropdown.
4013 """
4014 if not engine_options or not isinstance(settings_snapshot, dict):
4015 return
4016 try:
4017 from ...security.egress.validators import (
4018 guarded_engine_url_descriptors,
4019 )
4020 from ...security.egress.warnings import (
4021 check_private_engine_url_blocked,
4022 )
4023 from ...utilities.type_utils import unwrap_setting
4025 guarded_urls = {
4026 d.engine_name: d
4027 for d in guarded_engine_url_descriptors()
4028 if d.is_public
4029 }
4030 except Exception: # pragma: no cover - defensive
4031 logger.exception("guarded engine-URL overlay unavailable")
4032 return
4034 for opt in engine_options:
4035 # Everything that touches the option — including the id lookup and
4036 # the existing-decision read — runs inside the try, so a malformed
4037 # entry is swallowed per engine as the docstring promises rather
4038 # than aborting the overlay for the rest of the list.
4039 desc = None
4040 try:
4041 desc = guarded_urls.get(opt.get("value"))
4042 if desc is None:
4043 continue
4044 existing = opt.get("egress")
4045 if isinstance(existing, dict) and not existing.get("allowed", True):
4046 continue
4047 url_value = unwrap_setting(
4048 settings_snapshot.get(desc.url_setting, "")
4049 )
4050 if check_private_engine_url_blocked(
4051 desc.engine_name,
4052 desc.engine_name,
4053 desc.url_setting,
4054 url_value,
4055 True,
4056 False,
4057 ):
4058 opt["egress"] = {
4059 "allowed": False,
4060 "reason": "private_url_unapproved",
4061 }
4062 except Exception:
4063 engine_name = desc.engine_name if desc is not None else "unknown"
4064 logger.exception(
4065 f"guarded-URL overlay failed for engine {engine_name}"
4066 )
4069@router.get("/api/available-search-engines")
4070def api_get_available_search_engines(
4071 request: Request,
4072 username: Annotated[str, Depends(require_auth)],
4073):
4074 """Get available search engines.
4076 Optional query params (issue #5204):
4077 ``egress_scope`` — when set, each option gets an
4078 ``egress: {allowed, reason}`` field matching the same PDP the
4079 request-boundary precheck uses. The frontend uses this to disable
4080 (or hide) options that would be refused under the active scope.
4081 ``primary`` — the user's saved primary search tool, used by the
4082 PDP for strict / adaptive scope resolution.
4084 When ``egress_scope`` is absent the SCOPE filtering is skipped, but
4085 the guarded engine-URL overlay still runs: it stamps
4086 ``egress: {allowed: false, reason: "private_url_unapproved"}`` on any
4087 public engine whose configured instance URL the runtime will refuse
4088 for lack of operator approval. That guard ignores
4089 ``policy.egress_scope`` by design, so gating the overlay on the
4090 opt-in would have hidden it on every default (``adaptive``) install.
4091 Options that are not guard-blocked keep the historical shape (no
4092 ``egress`` key), so existing callers — the settings page, the news
4093 form — are unaffected except for the engines they could not have
4094 used anyway.
4095 """
4096 try:
4097 # Issue #5204: optional egress-scope filter. The scope/primary
4098 # query params opt INTO a per-option egress decision; without
4099 # them the response is the historical, unfiltered list.
4100 # Flask read these from request.args; the FastAPI equivalent is
4101 # request.query_params.
4102 requested_scope = (
4103 (request.query_params.get("egress_scope") or "").strip().lower()
4104 )
4105 requested_primary = (request.query_params.get("primary") or "").strip()
4106 apply_egress_filter = requested_scope in (
4107 "private_only",
4108 "public_only",
4109 )
4111 with get_user_db_session(username) as db_session:
4112 settings_manager = get_settings_manager(db_session, username)
4114 # Get search engines using the same approach as search_engines_config.py
4115 from ...web_search_engines.search_engines_config import (
4116 search_config,
4117 )
4118 from ...web_search_engines.engine_groups import (
4119 classify_engine_group,
4120 effective_group,
4121 group_label,
4122 group_order,
4123 )
4125 search_engines = search_config(
4126 username=username, db_session=db_session
4127 )
4129 # Get user's favorites using SettingsManager
4130 favorites = settings_manager.get_setting("search.favorites", [])
4131 if not isinstance(favorites, list): 4131 ↛ 4132line 4131 didn't jump to line 4132 because the condition on line 4131 was never true
4132 favorites = []
4134 # Extract search engines from config
4135 engines_dict = {}
4136 engine_options = []
4138 if search_engines: 4138 ↛ 4238line 4138 didn't jump to line 4238 because the condition on line 4138 was always true
4139 # Format engines for API response with metadata
4140 from ...security.module_whitelist import (
4141 get_safe_module_class,
4142 SecurityError,
4143 )
4145 for engine_id, engine_data in search_engines.items():
4146 if not engine_data.get("supports_query_search", True):
4147 continue
4148 # Try to load the engine class to get metadata
4149 engine_class = None
4150 try:
4151 module_path = engine_data.get("module_path")
4152 class_name = engine_data.get("class_name")
4153 if module_path and class_name:
4154 # Use secure whitelist-validated import
4155 engine_class = get_safe_module_class(
4156 module_path, class_name
4157 )
4158 except SecurityError:
4159 logger.warning(
4160 f"Security: Blocked unsafe module for {engine_id}"
4161 )
4162 except Exception as e:
4163 logger.debug(
4164 f"Could not load engine class for {engine_id}: {e}"
4165 )
4167 # Get icon and category from engine attributes
4168 icon, category = _get_engine_icon_and_category(
4169 engine_data, engine_class
4170 )
4172 # Check if engine requires an API key
4173 requires_api_key = engine_data.get(
4174 "requires_api_key", False
4175 )
4177 # Build display name with icon, category, and API key status
4178 base_name = engine_data.get("display_name", engine_id)
4179 if requires_api_key:
4180 label = f"{icon} {base_name} ({category}, API key)"
4181 else:
4182 label = f"{icon} {base_name} ({category}, Free)"
4184 # Check if engine is a favorite
4185 is_favorite = engine_id in favorites
4187 # Classify the engine into a selector band. base_group
4188 # ignores favorite status (used by the frontend to move an
4189 # engine back to its category when un-starred); the
4190 # effective band is the Favorites overlay when starred.
4191 # See engine_groups.py.
4192 base_group = classify_engine_group(
4193 engine_id, category, requires_api_key
4194 )
4195 shown_group = effective_group(base_group, is_favorite)
4197 engines_dict[engine_id] = {
4198 "display_name": base_name,
4199 "description": engine_data.get("description", ""),
4200 "strengths": engine_data.get("strengths", []),
4201 "icon": icon,
4202 "category": category,
4203 "requires_api_key": requires_api_key,
4204 "is_favorite": is_favorite,
4205 }
4207 engine_options.append(
4208 {
4209 "value": engine_id,
4210 "label": label,
4211 "icon": icon,
4212 "category": category,
4213 "requires_api_key": requires_api_key,
4214 "is_favorite": is_favorite,
4215 "group": shown_group,
4216 "group_label": group_label(shown_group),
4217 "group_order": group_order(shown_group),
4218 "base_group": base_group,
4219 "base_group_label": group_label(base_group),
4220 "base_group_order": group_order(base_group),
4221 # Surface the per-engine ``agent_enabled`` flag so
4222 # the frontend can disable engines the LangGraph
4223 # research agent hides from its specialized tool
4224 # list. Defaults to True so engines that don't
4225 # carry the flag (only ``collection_*`` sets it
4226 # explicitly) stay selectable — the frontend's
4227 # LangGraph-strategy check is the only consumer
4228 # and short-circuits for other strategies, so a
4229 # True default is safe everywhere.
4230 "agent_enabled": engine_data.get(
4231 "agent_enabled", True
4232 ),
4233 }
4234 )
4236 # Sort engine_options by band order (favorites band first), then
4237 # alphabetically by label within each band.
4238 engine_options.sort(
4239 key=lambda x: (
4240 x.get("group_order", 999),
4241 x.get("label", "").lower(),
4242 )
4243 )
4245 # If no engines found, log the issue but return empty list
4246 if not engine_options: 4246 ↛ 4247line 4246 didn't jump to line 4247 because the condition on line 4246 was never true
4247 logger.warning("No search engines found in configuration")
4249 # Issue #5204: when the caller opts in via ?egress_scope=&primary=,
4250 # stamp each option with the PDP's per-engine decision so the
4251 # frontend can disable/hide the ones that would be refused at
4252 # submit time. UNPROTECTED and missing-scope callers skip the
4253 # SCOPE filtering (the historical shape) but still get the
4254 # guarded engine-URL overlay: that guard is scope-independent by
4255 # design (see ``_apply_guarded_url_overlay``), so an engine the
4256 # runtime will refuse must read as disabled under the shipped
4257 # ``adaptive`` default too, not only under ``public_only``.
4258 # A snapshot failure must not be replaced with an empty policy:
4259 # the filter and the overlay would then deny nothing, so the
4260 # list would be served unfiltered. Let it reach the handlers
4261 # below, which return an error instead of a list.
4262 policy_snapshot = settings_manager.get_settings_snapshot()
4263 policy_snapshot = policy_snapshot or {}
4264 if apply_egress_filter:
4265 # Runs the guarded-URL overlay itself, after the PDP pass.
4266 _classify_options_for_egress(
4267 engine_options,
4268 egress_scope=requested_scope,
4269 primary_engine=requested_primary,
4270 settings_snapshot=policy_snapshot,
4271 username=username,
4272 search_engines=search_engines,
4273 )
4274 else:
4275 _apply_guarded_url_overlay(engine_options, policy_snapshot)
4277 return {
4278 "engines": engines_dict,
4279 "engine_options": engine_options,
4280 "favorites": favorites,
4281 }
4283 except SettingsValueError as error:
4284 return _settings_value_error_response("available search engines", error)
4285 except Exception:
4286 logger.exception("Error getting available search engines")
4287 return JSONResponse(
4288 {"error": "Failed to retrieve search engines"}, status_code=500
4289 )
4292@router.get("/api/search-favorites")
4293def api_get_search_favorites(
4294 request: Request,
4295 username: Annotated[str, Depends(require_auth)],
4296):
4297 """Get the list of favorite search engines for the current user"""
4298 try:
4299 with get_user_db_session(username) as db_session:
4300 settings_manager = get_settings_manager(db_session, username)
4301 favorites = settings_manager.get_setting("search.favorites", [])
4302 if not isinstance(favorites, list):
4303 favorites = []
4304 return {"favorites": favorites}
4306 except Exception:
4307 logger.exception("Error getting search favorites")
4308 return JSONResponse(
4309 {"error": "Failed to retrieve favorites"}, status_code=500
4310 )
4313@router.put("/api/search-favorites")
4314@settings_limit
4315async def api_update_search_favorites(
4316 request: Request,
4317 username: Annotated[str, Depends(require_auth)],
4318):
4319 """Update the list of favorite search engines for the current user"""
4320 data, body_error = await read_json_dict(
4321 request, "simple", "No data provided"
4322 )
4323 if body_error is not None:
4324 return body_error
4326 def _impl():
4327 try:
4328 favorites = data.get("favorites")
4329 if favorites is None:
4330 return JSONResponse(
4331 {"error": "No favorites provided"}, status_code=400
4332 )
4334 if not isinstance(favorites, list):
4335 return JSONResponse(
4336 {"error": "Favorites must be a list"}, status_code=400
4337 )
4339 with get_user_db_session(username) as db_session:
4340 settings_manager = get_settings_manager(db_session, username)
4341 if settings_manager.settings_locked:
4342 return JSONResponse(
4343 {"error": "Settings are locked"}, status_code=403
4344 )
4345 if settings_manager.set_setting("search.favorites", favorites):
4346 invalidate_settings_caches(username)
4347 return {
4348 "message": "Favorites updated successfully",
4349 "favorites": favorites,
4350 }
4352 return JSONResponse(
4353 {"error": "Failed to update favorites"}, status_code=500
4354 )
4356 except Exception:
4357 logger.exception("Error updating search favorites")
4358 return JSONResponse(
4359 {"error": "Failed to update favorites"}, status_code=500
4360 )
4362 # SQLCipher PBKDF2 key derivation in get_user_db_session blocks the
4363 # event loop for hundreds of ms on first call after login. Offload
4364 # the entire DB-touching body so concurrent requests don't serialise.
4365 return await run_db_sync(_impl)
4368@router.post("/api/search-favorites/toggle")
4369@settings_limit
4370async def api_toggle_search_favorite(
4371 request: Request,
4372 username: Annotated[str, Depends(require_auth)],
4373):
4374 """Toggle a search engine as favorite"""
4375 data, body_error = await read_json_dict(
4376 request, "simple", "No data provided"
4377 )
4378 if body_error is not None:
4379 return body_error
4381 def _impl():
4382 try:
4383 engine_id = data.get("engine_id")
4384 if not engine_id:
4385 return JSONResponse(
4386 {"error": "No engine_id provided"}, status_code=400
4387 )
4389 with get_user_db_session(username) as db_session:
4390 settings_manager = get_settings_manager(db_session, username)
4391 if settings_manager.settings_locked:
4392 return JSONResponse(
4393 {"error": "Settings are locked"}, status_code=403
4394 )
4396 # Get current favorites
4397 favorites = settings_manager.get_setting("search.favorites", [])
4398 if not isinstance(favorites, list):
4399 favorites = []
4400 else:
4401 # Make a copy to avoid modifying the original
4402 favorites = list(favorites)
4404 # Toggle the engine
4405 is_favorite = engine_id in favorites
4406 if is_favorite:
4407 favorites.remove(engine_id)
4408 is_favorite = False
4409 else:
4410 favorites.append(engine_id)
4411 is_favorite = True
4413 # Update the setting
4414 if settings_manager.set_setting("search.favorites", favorites):
4415 invalidate_settings_caches(username)
4416 return {
4417 "message": "Favorite toggled successfully",
4418 "engine_id": engine_id,
4419 "is_favorite": is_favorite,
4420 "favorites": favorites,
4421 }
4423 return JSONResponse(
4424 {"error": "Failed to toggle favorite"}, status_code=500
4425 )
4427 except Exception:
4428 logger.exception("Error toggling search favorite")
4429 return JSONResponse(
4430 {"error": "Failed to toggle favorite"}, status_code=500
4431 )
4433 return await run_db_sync(_impl)
4436# Legacy routes for backward compatibility - these will redirect to the new routes
4437@router.get("/main")
4438def main_config_page(
4439 request: Request, username: Annotated[str, Depends(require_auth)]
4440):
4441 """Redirect to app settings page"""
4442 return RedirectResponse(url="/settings/", status_code=302)
4445@router.get("/collections")
4446def collections_config_page(
4447 request: Request, username: Annotated[str, Depends(require_auth)]
4448):
4449 """Redirect to app settings page"""
4450 return RedirectResponse(url="/settings/", status_code=302)
4453@router.get("/api_keys")
4454def api_keys_config_page(
4455 request: Request, username: Annotated[str, Depends(require_auth)]
4456):
4457 """Redirect to LLM settings page"""
4458 return RedirectResponse(url="/settings/", status_code=302)
4461@router.get("/search_engines")
4462def search_engines_config_page(
4463 request: Request, username: Annotated[str, Depends(require_auth)]
4464):
4465 """Redirect to search settings page"""
4466 return RedirectResponse(url="/settings/", status_code=302)
4469@router.get("/llm")
4470def llm_config_page(
4471 request: Request, username: Annotated[str, Depends(require_auth)]
4472):
4473 """Redirect to LLM settings page"""
4474 return RedirectResponse(url="/settings/", status_code=302)
4477@router.post("/open_file_location")
4478def open_file_location(
4479 request: Request, username: Annotated[str, Depends(require_auth)]
4480):
4481 """Open the location of a configuration file.
4483 Security: This endpoint is disabled for server deployments.
4484 It only makes sense for desktop usage where the server and client are on the same machine.
4485 """
4486 return JSONResponse(
4487 {
4488 "status": "error",
4489 "message": "This feature is disabled. It is only available in desktop mode.",
4490 },
4491 status_code=403,
4492 )
4495# CSRF token is injected globally via fastapi_app template globals
4498@router.get("/api/warnings")
4499def api_get_warnings(
4500 request: Request, username: Annotated[str, Depends(require_auth)]
4501):
4502 """Get current warnings based on settings"""
4503 try:
4504 warnings = calculate_warnings(username=username)
4505 return {"warnings": warnings}
4506 except Exception:
4507 logger.exception("Error getting warnings")
4508 return JSONResponse(
4509 {"error": "Failed to retrieve warnings"}, status_code=500
4510 )
4513@router.get("/api/backup-status")
4514def api_get_backup_status(
4515 request: Request, username: Annotated[str, Depends(require_auth)]
4516):
4517 """Get backup status for the current user."""
4518 try:
4519 from ...config.paths import get_user_backup_directory
4521 if not username: 4521 ↛ 4522line 4521 didn't jump to line 4522 because the condition on line 4521 was never true
4522 return JSONResponse({"error": "Not authenticated"}, status_code=401)
4524 from ...utilities.formatting import human_size
4526 backup_dir = get_user_backup_directory(username)
4528 # Sort by modification time (not filename) for robustness
4529 backup_list = []
4530 total_size = 0
4531 from ...database.backup.backup_service import is_safe_glob_result
4533 for b in backup_dir.glob("ldr_backup_*.db"):
4534 # Symlink/traversal guard (main #4663): skip a planted symlink whose
4535 # target escapes the per-user backup dir, so its external file's
4536 # name/size/mtime can't be exfiltrated through this endpoint.
4537 if not is_safe_glob_result(b, backup_dir):
4538 continue
4539 try:
4540 stat = b.stat()
4541 total_size += stat.st_size
4542 backup_list.append(
4543 {
4544 "filename": b.name,
4545 "size_bytes": stat.st_size,
4546 "size_human": human_size(stat.st_size),
4547 "created_at": datetime.fromtimestamp(
4548 stat.st_mtime, tz=timezone.utc
4549 ).isoformat(),
4550 "_mtime": stat.st_mtime,
4551 }
4552 )
4553 except FileNotFoundError:
4554 continue
4556 # Sort newest first by mtime, then remove internal field
4557 backup_list.sort(key=lambda x: x["_mtime"], reverse=True)
4558 for entry in backup_list:
4559 del entry["_mtime"]
4561 backup_enabled = _get_setting_from_session(
4562 "backup.enabled", username, True
4563 )
4565 return {
4566 "enabled": bool(backup_enabled),
4567 "count": len(backup_list),
4568 "backups": backup_list,
4569 "total_size_bytes": total_size,
4570 "total_size_human": human_size(total_size),
4571 }
4573 except Exception:
4574 logger.exception("Error getting backup status")
4575 return JSONResponse(
4576 {"error": "Failed to retrieve backup status"}, status_code=500
4577 )
4580@router.get("/api/ollama-status")
4581def check_ollama_status(
4582 request: Request, username: Annotated[str, Depends(require_auth)]
4583):
4584 """Check if Ollama is running and available"""
4585 try:
4586 # Get Ollama URL from settings
4587 raw_base_url = _get_setting_from_session(
4588 "llm.ollama.url", username, DEFAULT_OLLAMA_URL
4589 )
4590 base_url = (
4591 normalize_url(raw_base_url) if raw_base_url else DEFAULT_OLLAMA_URL
4592 )
4594 response = safe_get(
4595 f"{base_url}/api/version",
4596 timeout=2,
4597 allow_localhost=True,
4598 allow_private_ips=True,
4599 )
4601 if response.status_code == 200:
4602 return {
4603 "running": True,
4604 "version": response.json().get("version", "unknown"),
4605 }
4606 return {
4607 "running": False,
4608 "error": f"Ollama returned status code {response.status_code}",
4609 }
4611 except requests.exceptions.RequestException:
4612 logger.exception("Ollama check failed")
4613 return {
4614 "running": False,
4615 "error": "Failed to check search engine status",
4616 }
4619@router.get("/api/rate-limiting/status")
4620def api_get_rate_limiting_status(
4621 request: Request, username: Annotated[str, Depends(require_auth)]
4622):
4623 """Get current rate limiting status and statistics"""
4624 try:
4625 # exploration_rate / learning_rate are the *configured* settings, not
4626 # the effective operating values: AdaptiveRateLimitTracker._apply_profile
4627 # scales them by the active profile (conservative ~0.5x/0.7x, aggressive
4628 # ~1.5x/1.3x, each capped). The profile is reported alongside so a
4629 # consumer can tell which transform the running tracker applies; we
4630 # deliberately don't duplicate that scaling math here.
4631 status = {
4632 # Default True to match the schema default (default_settings.json)
4633 # and the tracker's web-mode default; the prior endpoint reported
4634 # the tracker's effective enabled state, which was on by default.
4635 "enabled": _get_setting_from_session(
4636 "rate_limiting.enabled", username, True
4637 ),
4638 "profile": _get_setting_from_session(
4639 "rate_limiting.profile", username, "balanced"
4640 ),
4641 "exploration_rate": _get_setting_from_session(
4642 "rate_limiting.exploration_rate", username, 0.1
4643 ),
4644 "learning_rate": _get_setting_from_session(
4645 "rate_limiting.learning_rate", username, 0.45
4646 ),
4647 "memory_window": _get_setting_from_session(
4648 "rate_limiting.memory_window", username, 100
4649 ),
4650 }
4652 with get_user_db_session(username) as db_session:
4653 estimates = (
4654 db_session.query(RateLimitEstimate)
4655 .order_by(RateLimitEstimate.engine_type)
4656 .all()
4657 )
4659 engines = []
4660 for est in estimates:
4661 engines.append(
4662 {
4663 "engine_type": est.engine_type,
4664 "base_wait_seconds": round(est.base_wait_seconds, 2),
4665 "min_wait_seconds": round(est.min_wait_seconds, 2),
4666 "max_wait_seconds": round(est.max_wait_seconds, 2),
4667 "last_updated": est.last_updated,
4668 "total_attempts": est.total_attempts,
4669 "success_rate": round(est.success_rate * 100, 1),
4670 }
4671 )
4673 return {"status": status, "engines": engines}
4675 except Exception:
4676 logger.exception("Error getting rate limiting status")
4677 return JSONResponse(
4678 {"error": "An internal error occurred"}, status_code=500
4679 )
4682@router.post("/api/rate-limiting/engines/{engine_type}/reset")
4683def api_reset_engine_rate_limiting(
4684 request: Request,
4685 engine_type,
4686 username: Annotated[str, Depends(require_auth)],
4687):
4688 """Reset (forget) the learned rate-limit estimate for a specific engine.
4690 Deletes the engine's persisted ``RateLimitEstimate`` row from the user's
4691 database so the adaptive tracker re-learns it from scratch. The previous
4692 implementation called the per-request ``get_tracker()``, whose mutation
4693 path is gated on a research-session context that is absent in an analytics
4694 HTTP request — so it was a silent no-op that never cleared the persisted
4695 estimate the ``/status`` and ``/current`` endpoints display (#4721).
4696 """
4697 try:
4698 # (#6945) Same retry wrap the tracker's reset_engine got: a
4699 # transient SQLite writer-lock blip must not fail the request
4700 # on the first "database is locked". The closure opens a fresh
4701 # ``get_user_db_session`` per attempt so the writer lock is
4702 # released between tries; after retry exhaustion the existing
4703 # error handling below still applies.
4704 def _reset_db():
4705 with get_user_db_session(username) as db_session:
4706 db_session.query(RateLimitEstimate).filter_by(
4707 engine_type=engine_type
4708 ).delete(synchronize_session=False)
4709 db_session.commit()
4711 retry_on_db_lock(_reset_db)
4713 return {"message": f"Rate limiting data reset for {engine_type}"}
4715 except Exception:
4716 logger.exception(
4717 "Error resetting rate limiting for {}",
4718 _log_value_preview(engine_type),
4719 )
4720 return JSONResponse(
4721 {"error": "An internal error occurred"}, status_code=500
4722 )
4725def _cleanup_rate_limit_estimates_sync(username: str, cutoff: float) -> None:
4726 """Delete persisted rate-limit estimates last updated before *cutoff*."""
4728 # (#6945) Same retry wrap the tracker's cleanup_old_data got — see
4729 # api_reset_engine_rate_limiting above for the rationale. Fresh
4730 # session per attempt; lock exhaustion propagates to the route's
4731 # existing error handling.
4732 def _cleanup_db():
4733 with get_user_db_session(username) as db_session:
4734 db_session.query(RateLimitEstimate).filter(
4735 RateLimitEstimate.last_updated < cutoff
4736 ).delete(synchronize_session=False)
4737 db_session.commit()
4739 retry_on_db_lock(_cleanup_db)
4742@router.post("/api/rate-limiting/cleanup")
4743async def api_cleanup_rate_limiting(
4744 request: Request, username: Annotated[str, Depends(require_auth)]
4745):
4746 """Clean up old rate limiting data.
4748 Note: not using @require_json_body because the JSON body is optional
4749 here — the endpoint works with or without a payload (defaults to 30 days).
4751 Parse errors use the same response as a non-object body. The body is
4752 optional, so ``read_json_dict`` would reject valid ``null`` and cannot
4753 be used here. The isinstance check below has the same reasoning as
4754 ``@require_json_body``:
4755 a *valid but truthy non-dict* JSON body (e.g. ``[]`` or a bare number)
4756 would otherwise reach ``data.get(...)`` and raise ``AttributeError``.
4758 The parse catches exactly what ``read_json_dict`` does: ``ValueError``
4759 (bad syntax, undecodable bytes, an oversized integer literal) and
4760 ``RecursionError`` (nesting too deep to parse). Anything else, such as
4761 the request-size limit's exception, must propagate to its own handler.
4762 """
4763 content_type = request.headers.get("content-type", "")
4764 is_json = "application/json" in content_type
4765 try:
4766 data = await request.json() if is_json else None
4767 except (ValueError, RecursionError):
4768 return json_body_error("simple", "Request body must be a JSON object")
4769 if data is not None and not isinstance(data, dict):
4770 return json_body_error("simple", "Request body must be a JSON object")
4771 days = data.get("days", 30) if data is not None else 30
4773 try:
4774 days = int(days)
4775 except (TypeError, ValueError):
4776 return JSONResponse(
4777 {"error": "'days' must be an integer"}, status_code=400
4778 )
4779 if days < 1 or days > 365:
4780 return JSONResponse(
4781 {"error": "'days' must be between 1 and 365"}, status_code=400
4782 )
4784 try:
4785 # Delete persisted estimates not updated within the window. Mirrors the
4786 # read endpoints (#4721): operate on RateLimitEstimate rather than the
4787 # per-request get_tracker(), whose cleanup path is a no-op outside a
4788 # research-session context. last_updated is a unix timestamp (Float).
4789 cutoff = time.time() - days * 86400
4790 await run_db_sync(_cleanup_rate_limit_estimates_sync, username, cutoff)
4792 return {
4793 "message": f"Cleaned up rate limiting data older than {days} days"
4794 }
4796 except Exception:
4797 logger.exception("Error cleaning up rate limiting data")
4798 return JSONResponse(
4799 {"error": "An internal error occurred"}, status_code=500
4800 )
4803@router.get("/api/bulk")
4804def get_bulk_settings(
4805 request: Request, username: Annotated[str, Depends(require_auth)]
4806):
4807 """Get multiple settings at once for performance."""
4808 try:
4809 # Get requested settings from query parameters
4810 requested = request.query_params.getlist("keys[]")
4811 if not requested:
4812 # Default to common settings if none specified
4813 requested = [
4814 "llm.provider",
4815 "llm.model",
4816 "search.tool",
4817 "search.iterations",
4818 "search.questions_per_iteration",
4819 "search.search_strategy",
4820 "benchmark.evaluation.provider",
4821 "benchmark.evaluation.model",
4822 "benchmark.evaluation.temperature",
4823 "benchmark.evaluation.endpoint_url",
4824 ]
4826 # Fetch all settings at once
4827 from ...security.data_sanitizer import DataSanitizer
4829 result = {}
4830 for key in requested:
4831 try:
4832 value = _get_setting_from_session(key, username)
4833 # Redact secret values (main does this — this endpoint is an
4834 # exfiltration channel for plaintext API keys/tokens via
4835 # ?keys[]=...). 'exists' reflects the RAW value, not the sentinel.
4836 result[key] = {
4837 "value": DataSanitizer.redact_value(key, None, value),
4838 "exists": value is not None,
4839 }
4840 except Exception:
4841 logger.warning(
4842 "Error getting setting {}", _log_value_preview(key)
4843 )
4844 result[key] = {
4845 "value": None,
4846 "exists": False,
4847 "error": "Failed to retrieve setting",
4848 }
4850 return {"success": True, "settings": result}
4852 except Exception:
4853 logger.exception("Error getting bulk settings")
4854 return JSONResponse(
4855 {"success": False, "error": "An internal error occurred"},
4856 status_code=500,
4857 )
4860@router.get("/api/data-location")
4861def api_get_data_location(
4862 request: Request, username: Annotated[str, Depends(require_auth)]
4863):
4864 """Get information about data storage location and security"""
4865 try:
4866 # Get the data directory path
4867 data_dir = get_data_directory()
4868 # Get the encrypted databases path
4869 encrypted_db_path = get_encrypted_database_path()
4871 # LDR_DATA_DIR is the only variable that moves the data directory, so
4872 # it alone decides whether the location is custom.
4873 custom_data_dir = get_custom_data_directory()
4875 # Get platform-specific default location info
4876 platform_info = {
4877 "Windows": "C:\\Users\\Username\\AppData\\Local\\local-deep-research",
4878 "macOS": "~/Library/Application Support/local-deep-research",
4879 "Linux": "~/.local/share/local-deep-research",
4880 }
4882 # Current platform
4883 current_platform = platform.system()
4884 if current_platform == "Darwin":
4885 current_platform = "macOS"
4887 # Get SQLCipher settings from environment
4888 from ...database.sqlcipher_utils import get_sqlcipher_settings
4890 # Debug logging
4891 logger.info(f"db_manager type: {type(db_manager)}")
4892 logger.info(
4893 f"db_manager.has_encryption: {getattr(db_manager, 'has_encryption', 'ATTRIBUTE NOT FOUND')}"
4894 )
4896 cipher_settings = (
4897 get_sqlcipher_settings() if db_manager.has_encryption else {}
4898 )
4900 return {
4901 "data_directory": str(data_dir),
4902 "database_path": str(encrypted_db_path),
4903 "encrypted_database_path": str(encrypted_db_path),
4904 "is_custom": custom_data_dir is not None,
4905 "custom_env_var": "LDR_DATA_DIR",
4906 "custom_env_value": (
4907 str(custom_data_dir) if custom_data_dir is not None else None
4908 ),
4909 "platform": current_platform,
4910 "platform_default": platform_info.get(
4911 current_platform, str(data_dir)
4912 ),
4913 "platform_info": platform_info,
4914 "security_notice": {
4915 "encrypted": db_manager.has_encryption,
4916 "warning": "All data including API keys stored in the database are securely encrypted."
4917 if db_manager.has_encryption
4918 else "All data including API keys stored in the database are currently unencrypted. Please ensure appropriate file system permissions are set.",
4919 "recommendation": "Your data is protected with database encryption."
4920 if db_manager.has_encryption
4921 else "Consider using environment variables for sensitive API keys instead of storing them in the database.",
4922 },
4923 "encryption_settings": cipher_settings,
4924 }
4926 except Exception:
4927 logger.exception("Error getting data location information")
4928 return JSONResponse(
4929 {"error": "Failed to retrieve data location"}, status_code=500
4930 )
4933def _is_blank_service_url(value) -> bool:
4934 """True when a notification URL counts as unset.
4936 ``DataSanitizer._is_empty_value`` and ``NotificationManager`` both treat
4937 a whitespace-only ``notifications.service_url`` as unconfigured, so the
4938 test endpoint has to agree -- a stored ``" "`` is truthy, and without
4939 this it would be handed to Apprise verbatim.
4940 """
4941 if isinstance(value, str):
4942 return not value.strip()
4943 return not value
4946async def _notification_test_body(request: Request) -> dict:
4947 """Parse the test-url body and stash it for the rate-limit predicate.
4949 Runs as a route dependency, i.e. *before* slowapi's decorator wrapper,
4950 which is the only way ``_caller_supplied_notification_url`` (a
4951 synchronous callback) can see the request body at all.
4953 ``await request.json()`` is deliberately called here rather than inside
4954 the handler's broad ``except Exception`` so a malformed body reaches the
4955 app's registered ``json.JSONDecodeError`` -> 400 handler instead of
4956 being swallowed and reported as a 500 (see
4957 ``web/dependencies/json_body.py``). A valid but non-dict body (e.g. a
4958 bare number) is normalised to ``{}`` rather than raising, matching
4959 main's ``request.get_json(silent=True)`` shape.
4960 """
4961 data = await request.json()
4962 if not isinstance(data, dict):
4963 data = {}
4964 request.state.notification_test_payload = data
4965 return data
4968def _caller_supplied_notification_url(request: Request) -> bool:
4969 """True when the test-url request names its own destination.
4971 Rate-limit exemption predicate. Testing a URL the caller just typed is
4972 the case the endpoint exists to serve and stays unlimited. Falling back
4973 to the caller's STORED URL does not: that path is a zero-argument send
4974 trigger, so it is the one that gets a bucket.
4975 """
4976 payload = getattr(request.state, "notification_test_payload", None)
4977 if not isinstance(payload, dict): 4977 ↛ 4978line 4977 didn't jump to line 4978 because the condition on line 4977 was never true
4978 return False
4979 submitted = payload.get("service_url")
4980 if _is_blank_service_url(submitted):
4981 return False
4982 return submitted != DataSanitizer.REDACTION_TEXT
4985# Error strings ``NotificationService.test_service`` can return that name
4986# no destination: operator instructions and shape refusals whose text is
4987# fixed, or parameterised only with counts. Kept as prefixes because two of
4988# them continue with a count or a second sentence.
4989_HOST_FREE_NOTIFICATION_TEST_ERRORS = (
4990 # The outbound master switch is off; only an operator can change it.
4991 "Outbound notifications are disabled.",
4992 # The input does not partition into unambiguous entries.
4993 "Notification service URL could not be parsed unambiguously",
4994 # More entries than MAX_NOTIFICATION_TARGETS -- a count, not a host.
4995 "Too many notification targets",
4996 # Apprise declined to register the parsed entries.
4997 "Failed to add service URL",
4998 # The validator rejected the URL without a reason of its own.
4999 "Invalid notification service URL.",
5000 # Validation passed; delivery did not.
5001 "Failed to send test notification",
5002)
5005def _is_host_free_notification_error(error_text: object) -> bool:
5006 """True when a ``test_service`` error is safe to echo for a URL the
5007 caller never named.
5009 The stored-URL fallback tests a destination the caller does not have to
5010 name and, once the setting is redacted, cannot read back. An error that
5011 names it -- the validator's ``Blocked private/internal IP address:
5012 <host>`` and ``Blocked cloud-metadata / link-local IP address: <host>``
5013 -- would disclose the stored hostname/IP through the very endpoint that
5014 exists to keep the URL hidden.
5016 Matching an ALLOWLIST rather than denylisting those two prefixes keeps
5017 the failure closed: an error string added to ``test_service`` later is
5018 suppressed (the user loses detail, the log keeps it) instead of leaking
5019 by default. Static operator instructions carry no host and must still
5020 reach the UI -- "set LDR_NOTIFICATIONS_ALLOW_OUTBOUND=true" is useless
5021 in a log the user cannot read, and ``docs/NOTIFICATIONS.md`` promises
5022 it inline.
5023 """
5024 return isinstance(error_text, str) and error_text.startswith(
5025 _HOST_FREE_NOTIFICATION_TEST_ERRORS
5026 )
5029# Own bucket, not the shared "settings" one: this caps the stored-URL
5030# fallback without spending the quota a user needs for saving settings.
5031# Keyed per authenticated user (the branch convention for settings routes)
5032# rather than main's per-IP default -- the destination being spammed is the
5033# caller's own configured webhook.
5034notification_test_limit = limiter.shared_limit(
5035 SETTINGS_RATE_LIMIT,
5036 scope="notification_test",
5037 key_func=_user_key,
5038 exempt_when=_caller_supplied_notification_url,
5039)
5042@router.post("/api/notifications/test-url")
5043@notification_test_limit
5044async def api_test_notification_url(
5045 request: Request,
5046 data: Annotated[dict, Depends(_notification_test_body)],
5047 username: Annotated[str, Depends(require_auth)],
5048):
5049 """
5050 Test a submitted notification URL or the calling user's stored URL.
5052 When ``service_url`` is missing, blank, or the redaction sentinel, the
5053 authenticated user's stored ``notifications.service_url`` is used
5054 instead. Blank includes whitespace-only, matching
5055 ``DataSanitizer._is_empty_value`` and the notification manager, which
5056 both treat ``" "`` as unconfigured -- otherwise Apprise is handed
5057 literal whitespace. An unconfigured stored URL returns 400. Test
5058 notifications still use a temporary Apprise instance.
5060 Security note: this endpoint was deliberately unlimited, on the grounds
5061 that users need to test URLs while configuring notifications. That
5062 reasoning covers a caller who submits a URL, and that path is still
5063 exempt. It does not cover the stored-URL fallback: with no body, this
5064 becomes a zero-argument trigger that sends to a destination the caller
5065 never has to name, so an authenticated caller could loop on an empty
5066 body to spam their configured service. That path consumes a dedicated
5067 rate-limit bucket. ``require_auth`` still bounds the blast radius to the
5068 caller's own notification services.
5070 A wrong-typed or hostile ``service_url`` *value* flows unchanged into
5071 ``NotificationService.test_service`` / ``NotificationURLValidator``,
5072 which already reject it cleanly (non-string, unparsable, or
5073 private/loopback targets all return ``{"success": False, "error": ...}``
5074 rather than raising).
5075 """
5076 service_url = data.get("service_url")
5077 # Captured before the stored-URL fallback replaces ``service_url``:
5078 # the response shaping below branches on who chose the destination.
5079 caller_supplied = _caller_supplied_notification_url(request)
5080 if _is_blank_service_url(service_url) or (
5081 service_url == DataSanitizer.REDACTION_TEXT
5082 ):
5083 # Off-loop: ``_get_setting_from_session`` opens a SQLCipher session
5084 # (PBKDF2 key derivation + disk I/O) and constructs a
5085 # SettingsManager, both synchronous. Called inline from this
5086 # ``async def`` it stalls the event loop, and the server runs
5087 # single-worker, so every other in-flight request stalls with it.
5088 service_url = await run_db_sync(
5089 _get_setting_from_session,
5090 "notifications.service_url",
5091 username,
5092 default="",
5093 )
5094 if _is_blank_service_url(service_url):
5095 return JSONResponse(
5096 {"success": False, "error": "No notification URL configured"},
5097 status_code=400,
5098 )
5100 try:
5101 from ...notifications.service import NotificationService
5103 # Create notification service instance and test the URL.
5104 # No password/session needed - URL provided directly, no DB access.
5105 # test_service performs a synchronous Apprise network send (can
5106 # block for the full connect/read timeout against a slow or
5107 # unreachable endpoint) — run it off the event loop.
5108 import asyncio
5110 from ...settings.env_registry import get_env_setting
5112 # Honour the operator's env gating (main does this) — otherwise the
5113 # service defaults to outbound_allowed=False and the URL test can never
5114 # succeed even when the operator set LDR_NOTIFICATIONS_ALLOW_OUTBOUND.
5115 notification_service = NotificationService(
5116 allow_private_ips=bool(
5117 get_env_setting("notifications.allow_private_ips", False)
5118 ),
5119 outbound_allowed=bool(
5120 get_env_setting("notifications.allow_outbound", False)
5121 ),
5122 )
5123 result = await asyncio.to_thread(
5124 notification_service.test_service, service_url
5125 )
5127 # Only return expected fields to prevent information leakage. On
5128 # the caller-supplied path the validator detail is safe to echo —
5129 # the caller just typed that URL. The stored-URL fallback is
5130 # different: an error that names the destination would reveal the
5131 # stored hostname/IP, content the redacted API otherwise never
5132 # returns. Those go to the server log and come back generic; the
5133 # host-free operator instructions still come back inline, since
5134 # they say what the user has to do next.
5135 if caller_supplied:
5136 return {
5137 "success": result.get("success", False),
5138 "message": result.get("message", ""),
5139 "error": result.get("error", ""),
5140 }
5141 if result.get("success"):
5142 return {
5143 "success": result.get("success", False),
5144 "message": result.get("message", ""),
5145 "error": "",
5146 }
5147 error_text = result.get("error", "") or ""
5148 logger.warning(
5149 "Stored-URL notification test failed for user={!r}: {}",
5150 username,
5151 error_text or result.get("message", ""),
5152 )
5153 return {
5154 "success": False,
5155 "message": "",
5156 "error": (
5157 error_text
5158 if _is_host_free_notification_error(error_text)
5159 else (
5160 "Failed to test notification service. Check server "
5161 "logs for details."
5162 )
5163 ),
5164 }
5166 except Exception:
5167 logger.exception("Error testing notification URL")
5168 return JSONResponse(
5169 {
5170 "success": False,
5171 "error": "Failed to test notification service. Check logs for details.",
5172 },
5173 status_code=500,
5174 )
5177# =============================================================================
5178# Catch-all {key} routes — MUST be last so they don't shadow specific routes
5179# =============================================================================
5182@router.get("/api/{key}")
5183def api_get_db_setting(
5184 request: Request,
5185 key,
5186 username: Annotated[str, Depends(require_auth)],
5187):
5188 """Get a specific setting by key from DB, falling back to defaults.
5190 Secret values (API keys, passwords, OAuth tokens) are redacted with the
5191 '[REDACTED]' sentinel — main does this (DataSanitizer.redact_value) so a
5192 single authenticated GET can't exfiltrate a plaintext credential. Safe to
5193 round-trip: the save path treats the sentinel as a no-op, so re-saving a
5194 redacted value never overwrites the stored credential.
5195 """
5196 try:
5197 with get_user_db_session(username) as db_session:
5198 settings_manager = get_settings_manager(db_session, username)
5200 db_setting = (
5201 db_session.query(Setting).filter(Setting.key == key).first()
5202 )
5204 if db_setting:
5205 from ...security.data_sanitizer import DataSanitizer
5207 # Overlay any LDR_* env-var override BEFORE redaction, so a
5208 # redacted secret still reflects the effective (env) value
5209 # rather than the stale DB row — same order main used
5210 # (_shape_single_effective_metadata, then redact once).
5211 effective_value, effective_editable = _apply_env_override(
5212 settings_manager,
5213 key,
5214 db_setting.value,
5215 db_setting.editable,
5216 )
5218 value = DataSanitizer.redact_value(
5219 db_setting.key, db_setting.ui_element, effective_value
5220 )
5221 setting_data = {
5222 "key": db_setting.key,
5223 "value": value,
5224 "type": db_setting.type
5225 if isinstance(db_setting.type, str)
5226 else db_setting.type.value,
5227 "name": db_setting.name,
5228 "description": db_setting.description,
5229 "category": db_setting.category,
5230 "ui_element": db_setting.ui_element,
5231 "options": db_setting.options,
5232 "min_value": db_setting.min_value,
5233 "max_value": db_setting.max_value,
5234 "step": db_setting.step,
5235 "visible": db_setting.visible,
5236 "editable": effective_editable,
5237 }
5238 # Operator-gate the "unprotected" egress escape hatch and
5239 # the "filesystem" PDF-storage option out of their
5240 # respective options lists unless explicitly enabled, and
5241 # normalise egress-scope's displayed value (#5148 /
5242 # 87537d9ec / fb49985aa).
5243 return _shape_pdf_storage_mode_setting(
5244 key, _shape_egress_scope_setting(key, setting_data)
5245 )
5247 default_meta = settings_manager.default_settings.get(key)
5248 if default_meta:
5249 from ...security.data_sanitizer import DataSanitizer
5251 # Same env-var overlay as the DB branch above — a default-
5252 # only key (no DB row yet) can still be pinned via LDR_*.
5253 effective_value, effective_editable = _apply_env_override(
5254 settings_manager,
5255 key,
5256 default_meta.get("value"),
5257 default_meta.get("editable", True),
5258 )
5260 default_value = DataSanitizer.redact_value(
5261 key,
5262 default_meta.get("ui_element", "text"),
5263 effective_value,
5264 )
5265 setting_data = {
5266 "key": key,
5267 "value": default_value,
5268 "type": default_meta.get("type", "APP"),
5269 "name": default_meta.get("name", key),
5270 "description": default_meta.get("description"),
5271 "category": default_meta.get("category"),
5272 "ui_element": default_meta.get("ui_element", "text"),
5273 "options": default_meta.get("options"),
5274 "min_value": default_meta.get("min_value"),
5275 "max_value": default_meta.get("max_value"),
5276 "step": default_meta.get("step"),
5277 "visible": default_meta.get("visible", True),
5278 "editable": effective_editable,
5279 }
5280 return _shape_pdf_storage_mode_setting(
5281 key, _shape_egress_scope_setting(key, setting_data)
5282 )
5284 return JSONResponse(
5285 {"error": f"Setting not found: {key}"}, status_code=404
5286 )
5287 except SettingsValueError as error:
5288 return _settings_value_error_response("setting", error)
5289 except Exception:
5290 logger.exception("Error getting setting {}", _log_value_preview(key))
5291 return JSONResponse(
5292 {"error": "Failed to retrieve settings"}, status_code=500
5293 )
5296@router.put("/api/{key}")
5297@settings_limit
5298async def api_update_setting(
5299 request: Request,
5300 key,
5301 username: Annotated[str, Depends(require_auth)],
5302):
5303 """Update a setting"""
5304 data, body_error = await read_json_dict(
5305 request, "simple", "No data provided"
5306 )
5307 if body_error is not None:
5308 return body_error
5309 return await run_db_sync(_api_update_setting_sync, data, key, username)
5312def _api_update_setting_sync(data, key, username):
5313 try:
5314 # Key PRESENCE, not truthiness. An *absent* "value" is still a 400,
5315 # but an explicit JSON null must stay distinguishable from it:
5316 # embeddings.openai.chunk_size is the one registered setting whose
5317 # default IS null ("use the provider default"), and the read side
5318 # already supports it (#5963).
5319 if "value" not in data:
5320 return JSONResponse({"error": "No value provided"}, status_code=400)
5321 value = data["value"]
5322 is_openai_chunk_size = key == "embeddings.openai.chunk_size"
5324 with get_user_db_session(username) as db_session:
5325 # Environment-locked settings (LDR_* env var override) are
5326 # rejected up front, before any DB lookup. Ported from Flask's
5327 # api_update_setting early guard (web/routes/settings_routes.py).
5328 # Without this, the write is still correctly blocked further
5329 # down by set_setting()/create_or_update_setting()'s own
5330 # _is_environment_locked() check — but that failure was
5331 # indistinguishable from any other and fell through to a
5332 # generic 500 "Failed to update setting {key}" instead of the
5333 # 403 naming the lock. Lost diagnostic + wrong status code, not
5334 # a data-exposure bug (main's contract restored here).
5335 settings_manager = get_settings_manager(db_session, username)
5336 if is_operator_only_setting(key):
5337 return JSONResponse(
5338 {"error": f"Setting {key} is operator-only"},
5339 status_code=403,
5340 )
5341 if settings_manager.settings_locked:
5342 return JSONResponse(
5343 {"error": "Settings are locked"}, status_code=403
5344 )
5345 if settings_manager._is_environment_locked(
5346 key, "api_update_setting"
5347 ):
5348 return JSONResponse(
5349 {"error": f"Setting {key} is environment-locked"},
5350 status_code=403,
5351 )
5353 # embeddings.openai.chunk_size is a whole-number batch size. The
5354 # generic numeric validation below only checks type + min/max, so
5355 # a boolean (coerced to 1/0) or a non-integer float (5.7) would
5356 # silently persist and then fail inside the embedding provider at
5357 # request time. Validate ONCE, up front, against the REGISTERED
5358 # metadata — before the db_setting lookup, so the create path
5359 # (row missing after a DELETE) is guarded too, and so a drifted
5360 # row ui_element cannot change how the value is coerced (#5979).
5361 # A null is allowed through unvalidated: it is this setting's
5362 # registered default (#5963).
5363 if is_openai_chunk_size:
5364 registered_metadata = settings_manager.default_settings[key]
5365 if value is not None:
5366 raw_value = value
5367 value = coerce_setting_for_write(
5368 key=key,
5369 value=value,
5370 ui_element=str(registered_metadata["ui_element"]),
5371 )
5372 if (
5373 isinstance(raw_value, bool)
5374 or not isinstance(value, (int, float))
5375 or (isinstance(value, float) and not value.is_integer())
5376 or value < registered_metadata["min_value"]
5377 ):
5378 logger.warning(
5379 f"Validation failed for setting {key}: "
5380 "value must be a whole number at or above "
5381 f"{registered_metadata['min_value']}"
5382 )
5383 return JSONResponse(
5384 {"error": f"Invalid value for setting {key}"},
5385 status_code=400,
5386 )
5387 value = int(value)
5388 elif value is None:
5389 # Every other key keeps the pre-#5174 contract: an explicit
5390 # null is rejected exactly like an absent one.
5391 return JSONResponse(
5392 {"error": "No value provided"}, status_code=400
5393 )
5395 db_setting = (
5396 db_session.query(Setting).filter(Setting.key == key).first()
5397 )
5399 if db_setting:
5400 if not db_setting.editable:
5401 return JSONResponse(
5402 {"error": f"Setting {key} is not editable"},
5403 status_code=403,
5404 )
5406 # See ``_coerced_for_guard``: the sentinel guards must see a
5407 # JSON-string-encoded container coerced to its real
5408 # dict/list shape, or a masked container submitted in that
5409 # form (a supported shape for PUT .../<key> with
5410 # ``{"value": "<json string>"}``) sails past the container
5411 # arm entirely.
5412 _guard_value = _coerced_for_guard(
5413 db_setting.key, db_setting.ui_element, value
5414 )
5416 # The redaction sentinel is a no-op for any sensitive
5417 # setting, and an empty string is a no-op only for password
5418 # inputs (which render blank, so an untouched field must not
5419 # wipe the secret). Companion to the same guard in
5420 # save_all_settings/save_settings. The idempotent 200 keeps
5421 # client-side save indicators from erroring (#5960).
5422 if _is_secret_empty_noop(
5423 key, db_setting.ui_element, _guard_value, db_setting.value
5424 ):
5425 logger.debug(
5426 f"Skipping sensitive value write for {key} via "
5427 "api_update_setting (no-op)"
5428 )
5429 return {
5430 "message": (
5431 f"Setting {key} unchanged "
5432 "(sensitive value not overwritten)"
5433 )
5434 }
5436 # An embedded sentinel is a corrupted edit rather than an
5437 # untouched round-trip, so it is a hard error here (400)
5438 # while the exact-match case above stays an idempotent 200
5439 # no-op (#5947).
5440 if _embeds_redaction_sentinel(
5441 key, db_setting.ui_element, _guard_value, db_setting.value
5442 ):
5443 logger.warning(
5444 "Rejected redaction-sentinel value for {!r} via "
5445 "api_update_setting (user={!r})",
5446 key,
5447 username,
5448 )
5449 return JSONResponse(
5450 {
5451 "error": _container_rejection_error(
5452 db_setting.ui_element, _guard_value
5453 )
5454 },
5455 status_code=400,
5456 )
5458 if value is not None and not is_openai_chunk_size:
5459 # Persist the MERGED value when a masked container's
5460 # sentinel leaves resolved against the stored row (see
5461 # ``_reconciled_container_value_for_write``); an
5462 # ordinary write is returned unchanged.
5463 value, _is_reconciled_container = (
5464 _reconciled_container_value_for_write(
5465 value, _guard_value, db_setting.value
5466 )
5467 )
5469 # Coerce to the correct Python type before saving (e.g.
5470 # string "5" -> int 5 for a number setting). chunk_size
5471 # was already coerced and bounds-checked above against
5472 # the registry, which is authoritative for a setting's
5473 # type; the row is only data. A reconciled secret-
5474 # bearing container is persisted exactly as merged
5475 # instead -- coercing it through a str-typed ui_element
5476 # would flatten it to its Python repr and disclose a
5477 # just-restored secret as a plain string (#6201).
5478 if not _is_reconciled_container:
5479 # Infer from the raw JSON type before text coercion.
5480 # Otherwise a legacy text row turns False into the
5481 # truthy string "False" before set_setting can repair
5482 # its checkbox metadata.
5483 effective_ui_element = _infer_ui_element(
5484 value, db_setting.ui_element
5485 )
5486 value = coerce_setting_for_write(
5487 key=db_setting.key,
5488 value=value,
5489 ui_element=effective_ui_element,
5490 )
5492 # A reconciled secret-bearing container skips
5493 # ``validate_setting`` the same way it skips coercion
5494 # above. The load-bearing reason is TYPE REJECTION, not
5495 # logging: ``validate_setting`` runs the value through
5496 # ``get_typed_setting_value`` for the row's DECLARED
5497 # ui_element, so a dict/list on a "number"/"range" row
5498 # comes back "Value must be a number" and the merged
5499 # edit -- with the secret just restored into it -- is
5500 # lost. Its other former reason (that same path logged
5501 # the raw, secret-bearing value at WARNING) is closed
5502 # at the converter itself as of #6201 round 6, so it no
5503 # longer justifies anything on its own;
5504 # ``_coerced_for_guard``'s comment already says this.
5505 # ACCEPTED LOSS: skipping validation also skips the
5506 # options-membership / min-max / checkbox-type checks,
5507 # so e.g. a "select" row that already holds a container
5508 # accepts a merged container with no options check.
5509 # Bounded to rows already holding a container under a
5510 # scalar ui_element -- a user-constructed shape -- and
5511 # the merge itself already resolved every leaf against
5512 # the stored row (#6201).
5513 is_valid, error_message = (
5514 (True, None)
5515 if _is_reconciled_container
5516 else validate_setting(db_setting, value)
5517 )
5518 if not is_valid:
5519 logger.warning(
5520 f"Validation failed for setting {key}: "
5521 f"{error_message}"
5522 )
5523 return JSONResponse(
5524 {"error": f"Invalid value for setting {key}"},
5525 status_code=400,
5526 )
5527 # set_setting() re-infers the type of a text row from
5528 # the value, so a non-finite JSON number would be
5529 # stored as an invalid number row. Check the row it
5530 # will store.
5531 rejection = (
5532 None
5533 if _is_reconciled_container
5534 else _strict_read_rejection(
5535 key,
5536 value,
5537 _infer_ui_element(value, db_setting.ui_element),
5538 )
5539 )
5540 if rejection is not None:
5541 logger.warning(
5542 "Validation failed for setting {}: {}",
5543 _log_value_preview(key),
5544 rejection,
5545 )
5546 return JSONResponse(
5547 {
5548 "error": (
5549 f"Invalid value for setting {key}: "
5550 f"{rejection}"
5551 )
5552 },
5553 status_code=400,
5554 )
5556 # Cross-field egress-policy validation. The full-form saves run
5557 # these guards unconditionally for every key; this single-key
5558 # PUT previously restricted the call to a 4-key allowlist
5559 # (policy.egress_scope / search.tool /
5560 # llm.allowed_local_hostnames / policy.trusted_search_engines),
5561 # which let an SSRF-shaped value through on any key the
5562 # allowlist omitted — notably
5563 # search.engine.web.searxng.default_params.instance_url. Run
5564 # it for every key, matching main
5565 # (web/routes/settings_routes.py, 87537d9ec).
5566 _all_db_settings = {
5567 s.key: s for s in db_session.query(Setting).all()
5568 }
5569 _err = first_egress_validation_error(
5570 {key: value}, _all_db_settings
5571 )
5572 if _err is not None:
5573 logger.bind(policy_audit=True).warning(
5574 "egress-policy setting rejected at api_update_setting",
5575 key=key,
5576 reason=_err.get("error"),
5577 )
5578 return JSONResponse(
5579 {"error": _err["error"]}, status_code=400
5580 )
5582 success = set_setting(key, value, db_session=db_session)
5583 if success:
5584 invalidate_settings_caches(username)
5585 # A document_scheduler.* toggle (e.g.
5586 # sweep_library_collections or generate_rag) must take
5587 # effect without a re-login.
5588 reschedule_document_jobs_if_needed(username, [key])
5589 reschedule_zotero_jobs_if_needed(username, [key])
5590 response_data: dict[str, Any] = {
5591 "message": f"Setting {key} updated successfully"
5592 }
5594 if key in WARNING_AFFECTING_KEYS:
5595 warnings = calculate_warnings(username=username)
5596 response_data["warnings"] = warnings
5597 # Never the value: it may carry a secret (see #6201).
5598 logger.debug(
5599 f"Setting {key} changed, calculated {len(warnings)} warnings"
5600 )
5602 return response_data
5603 return JSONResponse(
5604 {"error": f"Failed to update setting {key}"},
5605 status_code=500,
5606 )
5607 # Registered settings may be absent after DELETE. Recreate them
5608 # from trusted default metadata (type/options/min_value/
5609 # max_value/step/ui_element/editable) instead of the
5610 # caller-supplied request body, so callers cannot replace those
5611 # by racing a delete+recreate. Building setting_dict from
5612 # data[...] alone (the prior behaviour here) let a DELETE+PUT
5613 # round trip silently drop min/max bounds and degrade a
5614 # "number" setting to "text", after which even the
5615 # properly-validating bulk path accepted out-of-range values.
5616 # Genuinely custom keys (no registered default) keep the
5617 # namespace-prefix contract below. Ported from main
5618 # (web/routes/settings_routes.py, 87537d9ec).
5619 default_meta = settings_manager.default_settings.get(key)
5620 if default_meta is not None:
5621 if not default_meta.get("editable", True): 5621 ↛ 5622line 5621 didn't jump to line 5622 because the condition on line 5621 was never true
5622 return JSONResponse(
5623 {"error": f"Setting {key} is not editable"},
5624 status_code=403,
5625 )
5626 default_ui = str(default_meta.get("ui_element", "text"))
5627 if value is not None:
5628 value = coerce_setting_for_write(
5629 key=key, value=value, ui_element=default_ui
5630 )
5631 _validation_setting = SimpleNamespace(
5632 key=key,
5633 ui_element=default_ui,
5634 options=default_meta.get("options"),
5635 min_value=default_meta.get("min_value"),
5636 max_value=default_meta.get("max_value"),
5637 )
5638 is_valid, error_message = validate_setting(
5639 _validation_setting, value
5640 )
5641 if not is_valid:
5642 logger.warning(
5643 "Validation failed for recreated setting {}: {}",
5644 key,
5645 error_message,
5646 )
5647 return JSONResponse(
5648 {"error": f"Invalid value for setting {key}"},
5649 status_code=400,
5650 )
5651 setting_dict = dict(default_meta)
5652 setting_dict.update({"key": key, "value": value})
5653 default_type = setting_dict.get("type")
5654 if ( 5654 ↛ 5731line 5654 didn't jump to line 5731 because the condition on line 5654 was always true
5655 isinstance(default_type, str)
5656 and default_type in SettingType.__members__
5657 ):
5658 setting_dict["type"] = SettingType[default_type]
5659 else:
5660 # Namespace validation: reject new keys outside allowed
5661 # prefixes.
5662 if not _is_allowed_new_setting_key(key):
5663 logger.warning(
5664 "Security: Rejected setting outside allowed "
5665 "namespaces: {} (user={!r})",
5666 _log_value_preview(key),
5667 username,
5668 )
5669 return JSONResponse(
5670 {"error": _new_key_rejection_reason(key)},
5671 status_code=400,
5672 )
5674 setting_dict = {
5675 "key": key,
5676 "value": value,
5677 "name": key.split(".")[-1].replace("_", " ").title(),
5678 "description": f"Setting for {key}",
5679 }
5681 # Add additional metadata if provided.
5682 # 'visible' and 'editable' are system-controlled — not
5683 # accepted from callers.
5684 for field in [
5685 "type",
5686 "name",
5687 "description",
5688 "category",
5689 "ui_element",
5690 "options",
5691 "min_value",
5692 "max_value",
5693 "step",
5694 ]:
5695 if field in data:
5696 setting_dict[field] = data[field]
5698 # New keys have no registered metadata. Preserve JSON scalar
5699 # types so a false API toggle is stored as a checkbox, not a
5700 # text row whose "False" value remains truthy on reads.
5701 if "ui_element" not in setting_dict:
5702 setting_dict["ui_element"] = _infer_ui_element(value)
5704 # The caller may choose the type. An unsupported type, or a
5705 # value that type cannot convert, would make every
5706 # effective settings read of this account fail, and only a
5707 # DELETE could undo it.
5708 rejection = _strict_read_rejection(
5709 key, value, setting_dict["ui_element"]
5710 )
5711 if rejection is not None:
5712 logger.warning(
5713 "Rejected new setting {}: {}",
5714 _log_value_preview(key),
5715 rejection,
5716 )
5717 return JSONResponse(
5718 {
5719 "error": (
5720 f"Invalid value for setting {key}: {rejection}"
5721 )
5722 },
5723 status_code=400,
5724 )
5726 # Creation has no prior value, so the sentinel cannot mean
5727 # "keep the stored secret" the way it does on the update path —
5728 # every occurrence of it, exact match included, is a corrupted
5729 # client value that would be stored verbatim as the credential
5730 # (#5947).
5731 _create_ui = setting_dict.get("ui_element")
5732 # This is the one create branch that never coerces before its
5733 # guard: the REGISTERED-default branch above already coerces
5734 # `value` in place before reaching here, but a genuinely new
5735 # (unregistered) key has no coercion step of its own. Without
5736 # this, a masked container submitted as its JSON-string
5737 # encoding -- caught for a registered key -- would sail past
5738 # the container arm here the same way it did on the update
5739 # paths (see ``_coerced_for_guard``). Only takes effect when
5740 # the caller explicitly provided ``ui_element`` in the request
5741 # body (custom keys have none registered to infer it from).
5742 _create_guard_value = _coerced_for_guard(
5743 key,
5744 _create_ui if isinstance(_create_ui, str) else None,
5745 value,
5746 )
5747 if _embeds_sentinel_on_create(key, _create_ui, _create_guard_value):
5748 logger.warning(
5749 "Rejected redaction-sentinel value for {} via "
5750 "api_update_setting create (user={!r})",
5751 _log_value_preview(key),
5752 username,
5753 )
5754 return JSONResponse(
5755 {
5756 "error": _container_rejection_error(
5757 _create_ui if isinstance(_create_ui, str) else None,
5758 _create_guard_value,
5759 )
5760 },
5761 status_code=400,
5762 )
5764 # Apply egress validation to creation as well as updates.
5765 # Otherwise DELETE followed by PUT recreates a governed key with
5766 # no guards: `llm.allowed_local_hostnames` is `editable`, so the
5767 # delete endpoint accepts it, and `llm.` is an allowed prefix, so
5768 # re-creation passes the namespace check. A public host smuggled
5769 # into that key is then read into EgressContext.local_hostnames
5770 # and classified LOCAL, laundering it past private_only and
5771 # require_local_llm.
5772 #
5773 # Ported from main (web/routes/settings_routes.py), which runs
5774 # this at FOUR sites; this port had three. The update branch ~40
5775 # lines above already carries the guard and its comment claims
5776 # the hole is closed — it was closed for update only.
5777 _all_db_settings = {
5778 s.key: s for s in db_session.query(Setting).all()
5779 }
5780 _err = first_egress_validation_error({key: value}, _all_db_settings)
5781 if _err is not None:
5782 logger.bind(policy_audit=True).warning(
5783 "egress-policy setting rejected at api_update_setting create",
5784 key=key,
5785 reason=_err.get("error"),
5786 )
5787 return JSONResponse({"error": _err["error"]}, status_code=400)
5789 db_setting = create_or_update_setting(
5790 setting_dict, db_session=db_session
5791 )
5793 if db_setting:
5794 invalidate_settings_caches(username)
5795 reschedule_document_jobs_if_needed(username, [key])
5796 reschedule_zotero_jobs_if_needed(username, [key])
5797 from ...security.data_sanitizer import DataSanitizer
5799 return JSONResponse(
5800 {
5801 "message": f"Setting {key} created successfully",
5802 "setting": {
5803 "key": db_setting.key,
5804 # Don't echo a freshly-created password back in
5805 # plaintext — redact like every other settings
5806 # response that ships to the browser.
5807 #
5808 # Route this through ``redact_value`` rather
5809 # than ``is_sensitive_setting`` (#6201, R4).
5810 # ``is_sensitive_setting`` is a NAME predicate
5811 # with no type guard; ``redact_value`` adds the
5812 # guard every read path already applies, so a
5813 # checkbox/number row whose name merely matches
5814 # the broadened snake_case arm (e.g.
5815 # ``llm.x_supports_api_key``) keeps its typed
5816 # value instead of echoing the string
5817 # "[REDACTED]" that no GET of the same row
5818 # would ever return. It also masks nested
5819 # string leaves inside a created container,
5820 # which the bare sentinel substitution could
5821 # not express at all.
5822 "value": DataSanitizer.redact_value(
5823 key, db_setting.ui_element, db_setting.value
5824 ),
5825 "type": db_setting.type.value,
5826 "name": db_setting.name,
5827 },
5828 },
5829 status_code=201,
5830 )
5831 return JSONResponse(
5832 {"error": f"Failed to create setting {key}"},
5833 status_code=500,
5834 )
5835 except Exception:
5836 logger.exception("Error updating setting {}", _log_value_preview(key))
5837 return JSONResponse(
5838 {"error": "Failed to update setting"}, status_code=500
5839 )
5842@router.delete("/api/{key}")
5843@settings_limit
5844def api_delete_setting(
5845 request: Request,
5846 key,
5847 username: Annotated[str, Depends(require_auth)],
5848):
5849 """Delete a setting"""
5850 try:
5851 with get_user_db_session(username) as db_session:
5852 settings_manager = get_settings_manager(db_session, username)
5853 if is_operator_only_setting(key):
5854 return JSONResponse(
5855 {"error": f"Setting {key} is operator-only"},
5856 status_code=403,
5857 )
5859 # Settings lock (app.lock_settings) -- ported from main's
5860 # api_delete_setting (#5659, "enforce the settings lock on delete,
5861 # import and reset"). SettingsManager already refuses when
5862 # locked, so the write cannot happen either way; this repeats
5863 # the check at the route so a locked instance answers 403
5864 # rather than 200 with nothing written, which is what main's
5865 # own comment gives as the reason. Without it the merge that
5866 # brought #5659 in would have silently dropped the fix, since
5867 # it landed in a Flask file this migration deletes.
5868 if settings_manager.settings_locked:
5869 return JSONResponse(
5870 {"error": "Settings are locked"},
5871 status_code=403,
5872 )
5874 # Environment-locked settings (LDR_* env var override) are
5875 # rejected up front — see the matching guard in
5876 # _api_update_setting_sync above for the full rationale. Checked
5877 # before the existence lookup so an env-locked key still reports
5878 # 403 (not 404) even if a client races a delete against a key
5879 # that hasn't been materialized into a DB row yet, matching
5880 # main's contract (web/routes/settings_routes.py).
5881 if settings_manager._is_environment_locked(
5882 key, "api_delete_setting"
5883 ):
5884 return JSONResponse(
5885 {"error": f"Setting {key} is environment-locked"},
5886 status_code=403,
5887 )
5889 db_setting = (
5890 db_session.query(Setting).filter(Setting.key == key).first()
5891 )
5892 if not db_setting:
5893 return JSONResponse(
5894 {"error": f"Setting not found: {key}"}, status_code=404
5895 )
5897 if not db_setting.editable:
5898 return JSONResponse(
5899 {"error": f"Setting {key} is not editable"},
5900 status_code=403,
5901 )
5903 success = settings_manager.delete_setting(key)
5904 if success:
5905 invalidate_settings_caches(username)
5906 reschedule_document_jobs_if_needed(username, [key])
5907 reschedule_zotero_jobs_if_needed(username, [key])
5908 return {"message": f"Setting {key} deleted successfully"}
5909 return JSONResponse(
5910 {"error": f"Failed to delete setting {key}"}, status_code=500
5911 )
5912 except Exception:
5913 logger.exception("Error deleting setting {}", _log_value_preview(key))
5914 return JSONResponse(
5915 {"error": "Failed to delete setting"}, status_code=500
5916 )