Coverage for src/local_deep_research/security/data_sanitizer.py: 99%
125 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Security module for sanitizing sensitive data from data structures.
3This module ensures that sensitive information like API keys, passwords, and tokens
4are not accidentally leaked in logs, files, or API responses.
6Includes helpers for filtering research metadata in API responses to prevent
7settings_snapshot (which contains all application settings including API keys)
8from being sent to the frontend.
9"""
11import json
12from typing import Any, Set
14from ..error_handling.error_messages import get_known_research_error_message
17# The placeholder a redacted value is replaced with. Single source of truth
18# so that write-back guards (which must treat this sentinel as a no-op to
19# avoid persisting it over a real secret on a redacted GET round-trip)
20# cannot drift from what the redactor actually emits.
21REDACTION_TEXT = "[REDACTED]"
23# Older research rows persisted str(exception) in research_meta["error"].
24# That value can include report paths and other server-side details. API
25# responses retain only known authored messages; other text is replaced.
26PUBLIC_RESEARCH_ERROR = "Research failed. Check the server logs for details."
28# Unicode 16.0 DerivedCoreProperties.txt, Default_Ignorable_Code_Point.
29# Most Unicode Default_Ignorable_Code_Point characters are non-printable and
30# are removed by str.isprintable(). These exceptions are printable combining
31# marks or letter-category fillers, so list them explicitly. Do not drop Mn as
32# a category: ordinary combining marks are visible parts of legitimate names.
33_PRINTABLE_DEFAULT_IGNORABLES = (
34 frozenset({0x034F}) # COMBINING GRAPHEME JOINER
35 | frozenset(range(0x115F, 0x1161)) # Hangul fillers
36 | frozenset(range(0x17B4, 0x17B6)) # Khmer inherent vowels
37 | frozenset(range(0x180B, 0x1810)) # Mongolian variation separators
38 | frozenset({0x3164, 0xFFA0}) # Hangul fillers
39 | frozenset(range(0xFE00, 0xFE10)) # Variation Selectors
40 | frozenset(range(0xE0100, 0xE01F0)) # Variation Selectors Supplement
41)
43# Printable characters that Unicode specifies as rendering blank but that are
44# not default ignorables. They pass str.isprintable() and setting-key validation,
45# so they can otherwise disguise a sensitive leaf in the bulk settings endpoint.
46_PRINTABLE_BLANKS = frozenset(
47 {
48 0x2800, # BRAILLE PATTERN BLANK
49 0x13441, # EGYPTIAN HIEROGLYPH FULL BLANK
50 0x13442, # EGYPTIAN HIEROGLYPH HALF BLANK
51 0x1D159, # MUSICAL SYMBOL NULL NOTEHEAD
52 }
53)
56def _visible_leaf(key: str) -> str:
57 """The key's last dotted segment, normalized for the sensitive-name check.
59 Defense in depth for the bulk-settings GET, which lacks the
60 authoritative ``ui_element == "password"`` signal and falls back to the key
61 name. Drops non-printable chars (control, format \u2014 zero-width/BOM/bidi/tag
62 chars/soft hyphen \u2014 and non-ASCII spaces) plus printable Unicode default
63 ignorables (variation selectors, combining grapheme joiner, and fillers)
64 and explicitly known printable blank characters, then strips ASCII space
65 and lowercases. Ordinary visible combining marks are retained. This
66 intentionally fails closed: removing invisible padding can collapse a
67 distinct key onto a sensitive name and over-redact its value, which is
68 safer than exposing a potentially disguised secret.
69 """
70 leaf = key.rsplit(".", 1)[-1]
71 leaf = "".join(
72 ch
73 for ch in leaf
74 if ch.isprintable()
75 and ord(ch) not in _PRINTABLE_DEFAULT_IGNORABLES
76 and ord(ch) not in _PRINTABLE_BLANKS
77 )
78 return leaf.strip().lower()
81def _is_empty_value(value: Any) -> bool:
82 """True when a value counts as unconfigured for the redaction empty rule.
84 Covers ``None``, ``""``, ``[]``, ``{}`` plus whitespace-only strings:
85 the notification manager already treats ``" "`` as unconfigured
86 (``not service_urls.strip()``), so redacting it to the sentinel would
87 make an unset URL LOOK configured. Everything else (0, False,
88 non-empty strings, non-empty containers) counts as set.
89 """
90 if isinstance(value, str):
91 return not value.strip()
92 return value in (None, "", [], {})
95# Leading qualifiers that mark a leaf as a boolean FLAG *about* a secret rather
96# than the secret itself. ``search.engine.web.brave.requires_api_key`` is a
97# checkbox saying an engine needs a key; masking it would ship "[REDACTED]" to
98# the settings UI in place of true/false and make the write-back no-op guards
99# refuse legitimate writes. This vocabulary is retained for documentation and
100# so a future confirmed shape can be added deliberately; only the single
101# hardcoded leaf below is actually exempted (see ``_has_non_secret_qualifier``).
102_NON_SECRET_LEAF_PREFIXES = (
103 "allow_",
104 "allows_",
105 "disable_",
106 "enable_",
107 "enabled_",
108 "has_",
109 "is_",
110 "need_",
111 "needs_",
112 "require_",
113 "requires_",
114 "show_",
115 "skip_",
116 "support_",
117 "supports_",
118 "use_",
119 "uses_",
120)
122# The ONE confirmed, load-bearing qualifier+suffix shape: used verbatim as a
123# config dict key throughout the search-engine factories
124# (``tests/security/test_factory_secret_redaction.py``,
125# ``tests/security/test_noncollection_agent_enabled.py``) and shipped as the
126# last dotted segment of ``search.engine.web.*.requires_api_key`` for every
127# engine that needs one (``defaults/settings_*.json``).
128_EXEMPT_QUALIFIER_LEAF = "requires_api_key"
131def _has_non_secret_qualifier(leaf: str) -> bool:
132 """True only for the ONE confirmed, load-bearing exemption: ``leaf`` is
133 literally ``requires_api_key``.
135 An earlier version exempted ANY leaf STARTING WITH ANY
136 ``_NON_SECRET_LEAF_PREFIXES`` qualifier, combined with ANY sensitive
137 suffix from ``DEFAULT_SENSITIVE_KEYS`` -- a combinatorial exemption this
138 module's own docstrings never actually claimed ("only the exact
139 ``requires_api_key`` shape ... is a carve-out") but the code granted
140 anyway. That let a dotted key whose LAST SEGMENT genuinely starts with a
141 qualifier immediately followed by an unrelated sensitive suffix --
142 ``rag.skip_password``, ``llm.x.use_auth_token`` -- ship a real string
143 credential in the clear, purely because its shape happened to resemble
144 the one verified flag. Nothing confirms ``skip_password`` or
145 ``use_auth_token`` are booleans rather than a badly-named credential the
146 way ``requires_api_key`` is verified to be.
148 An even earlier version matched the qualifier at ANY underscore
149 boundary (``f"_{prefix}" in leaf``), not just the start; that was fixed
150 first (see ``test_bulk_get_qualifier_mid_key_no_longer_exempts_a_real_secret``)
151 and is a separate, already-closed gap from the one this narrows.
153 Audit scope (post-#6201 review, item d -- corrected): the loader
154 (``SettingsManager.default_settings``, ``settings/manager.py``) does
155 ``defaults_path.rglob("*.json")``, which walks every subdirectory --
156 not just the top-level ``defaults/settings_*.json`` files. The 478-key
157 figure this docstring previously cited was only ``default_settings.json``
158 (341 keys) plus the top-level ``settings_*.json`` files (137 keys); it
159 omitted three subdirectories the loader also reads
160 (``llm_providers/``, ``research_library/``, ``settings/search_engines/``,
161 108 more keys). Audited again against the FULL rglob'd scope -- 586
162 keys as of this writing -- narrowing from the qualifier-prefix list to
163 this single hardcoded leaf still changes zero shipped keys'
164 classification, confirming the prefix list was never doing anything
165 beyond gating this one shape in practice, over the scope the loader
166 actually reads rather than a subset of it. A boolean/int value stays
167 protected regardless of this function's answer -- ``redact_value``'s
168 type guard leaves a bool/int/None leaf untouched on the broadened arm
169 (see ``_is_exact_sensitive_match``).
171 Blast radius, precisely (post-#6201 review, item R3): this changes
172 behavior for a STRING value stored under a leaf shaped like this, AND
173 for a dict/list one. A container that matches ONLY the broadened arm
174 does not hit the type guard -- it takes the ``_force_redact_strings``
175 branch instead, which masks every STRING leaf inside it regardless of
176 the sub-key's name. Executed: with ``llm.x.use_auth_token`` exempted,
177 ``{"a": "hello"}`` stays ``{"a": "hello"}``; without the exemption it
178 becomes ``{"a": "[REDACTED]"}``. So a future addition to this
179 vocabulary widens or narrows redaction for strings and for the string
180 leaves nested inside containers -- only the bool/int/None half is
181 genuinely unaffected.
182 """
183 return leaf == _EXEMPT_QUALIFIER_LEAF
186def _matches_sensitive_name(leaf: str, sensitive_names: Set[str]) -> bool:
187 """True when a normalized leaf names a secret.
189 Two arms:
191 1. Exact match (``llm.openai.api_key`` -> ``api_key``).
192 2. Underscore-boundary suffix match. Settings keys use two separator
193 conventions: dotted (``llm.openai.api_key``) and flat snake_case
194 (``local_search_milvus_token``). A flat key has no dots, so its dotted
195 "leaf" is the entire key and arm 1 can never match it — which is how a
196 password-typed setting shipped in the clear from the bulk settings GET
197 (#5762). Note the miss was never specific to ``token``:
198 ``local_search_milvus_api_key`` or ``some_password`` would have evaded
199 arm 1 just as completely.
201 Arm 2 skips a leaf that IS the ``requires_api_key`` shape (see
202 ``_has_non_secret_qualifier``), so ``requires_api_key`` stays readable
203 while ``milvus_api_key`` does not. This carve-out is a single hardcoded
204 leaf, not a qualifier-prefix-plus-any-suffix combination: a namespaced
205 flat key such as ``llm_requires_api_key``, a key where the qualifier
206 word appears anywhere else before an unrelated sensitive suffix
207 (``local_search_use_milvus_token``), and a DIFFERENT qualifier+suffix
208 pairing that merely resembles the shape (``rag.skip_password``,
209 ``llm.x.use_auth_token``) are all NOT exempted -- only the exact
210 ``requires_api_key`` shape (as a whole key, or as a dotted key's last
211 segment) is a confirmed, load-bearing carve-out. Plural token counts
212 (``max_tokens``, ``supports_max_tokens``) end in ``_tokens``, a
213 different suffix than ``_token``, and stay readable too.
214 """
215 if leaf in sensitive_names:
216 return True
217 if _has_non_secret_qualifier(leaf):
218 return False
219 return any(leaf.endswith(f"_{name}") for name in sensitive_names)
222def _is_exact_sensitive_match(
223 key: str, ui_element: str | None, sensitive_names: Set[str]
224) -> bool:
225 """True under the ORIGINAL (pre-#5771) sensitivity rule: ``ui_element ==
226 "password"`` or an exact dotted-leaf match — i.e. everything
227 ``is_sensitive_setting`` matched before the broadened underscore-suffix
228 arm (arm 2 of ``_matches_sensitive_name``) was added.
230 ``redact_value`` uses this to scope its non-string type guard to only
231 the NEW broadened match. The exact/password arms have masked non-string
232 values (e.g. a boolean under a key literally named ``api_key``) since
233 before this PR; that established behavior is intentionally left as-is.
234 The broadened arm is a lexical suffix heuristic backed by a
235 single literal ``requires_api_key`` exemption, so a boolean/number
236 setting whose name happens to lexically match a sensitive suffix
237 (for example, a future ``verify_token``-style flag) must NOT be redacted: replacing a bool/int
238 with the "[REDACTED]" string corrupts it in the settings UI, which
239 only special-cases real booleans when rendering a checkbox
240 (``web/static/js/components/settings.js``), and silently writes back
241 ``false``/empty on the next save.
242 """
243 if ui_element == "password": 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true
244 return True
245 raw_leaf = key.rsplit(".", 1)[-1].lower()
246 return raw_leaf in sensitive_names or _visible_leaf(key) in sensitive_names
249def _force_redact_strings(value: Any, redaction_text: str) -> Any:
250 """Mask every string leaf inside a container, regardless of its own
251 sub-key name, while leaving bool/int/None leaves untouched.
253 Used when a KEY matches the broadened suffix-only sensitive arm (arm 2
254 of ``_matches_sensitive_name``) and its VALUE is a dict/list rather than
255 a plain string. ``redact_value``'s normal recursion masks by SUB-KEY
256 name, so a secret nested under a non-sensitive sub-key -- e.g.
257 ``{"value": "s3cr3t"}`` under a ``milvus_token`` setting -- would
258 otherwise survive unredacted: ``value`` is not itself a sensitive leaf
259 name. This walks the container and replaces every non-empty string leaf
260 with the sentinel unconditionally, so the secret cannot hide behind an
261 innocuous sub-key. It preserves round 2's guarantee that a bool/int
262 leaf is never corrupted into a string, and leaves empty/whitespace-only
263 string leaves readable so an unset nested field doesn't look configured
264 (matching ``_is_empty_value``'s rule for the top-level value).
265 """
266 if isinstance(value, str):
267 return redaction_text if value.strip() else value
268 if isinstance(value, dict):
269 return {
270 sub_key: _force_redact_strings(sub_val, redaction_text)
271 for sub_key, sub_val in value.items()
272 }
273 if isinstance(value, list):
274 return [_force_redact_strings(item, redaction_text) for item in value]
275 return value
278class DataSanitizer:
279 """Utility class for removing sensitive information from data structures."""
281 # Public alias of the module-level sentinel (see REDACTION_TEXT above).
282 REDACTION_TEXT: str = REDACTION_TEXT
284 # Default set of sensitive key names to redact
285 DEFAULT_SENSITIVE_KEYS: Set[str] = {
286 "api_key",
287 "apikey",
288 "password",
289 "secret",
290 "access_token",
291 "refresh_token",
292 "private_key",
293 "auth_token",
294 "session_token",
295 "csrf_token",
296 # Additional unambiguous secret leaf-names. The predicate is
297 # exact-match on the last dotted segment, so these matter especially
298 # for the bulk settings GET, which can only use the key-name heuristic
299 # (it passes no ui_element). All are unambiguously secrets and no
300 # current setting key uses them for non-secret data.
301 "client_secret",
302 "secret_key",
303 "bearer_token",
304 "api_secret",
305 "app_secret",
306 # The notification service URL (apprise-style) embeds credentials --
307 # e.g. mailto://user:pass@host, discord://webhook_id/token, Slack/ntfy
308 # tokens. The app already masks it in logs (mask_sensitive_url); this
309 # keeps it out of the settings API read paths too. Only
310 # notifications.service_url has this leaf, so no other setting is
311 # affected. The empty-value rule keeps an unconfigured URL readable.
312 "service_url",
313 # A bare "token" leaf. The qualified spellings above (access_token,
314 # auth_token, bearer_token, ...) left the unqualified name uncovered,
315 # so a setting keyed "<prefix>_token" with no dots in it (its leaf is
316 # the whole key) matched nothing and shipped in the clear from the
317 # bulk settings GET (#5762). "token" and "api_token" name a secret in
318 # every credential convention we ship; token COUNTS are spelled
319 # max_tokens / context_tokens / supports_max_tokens, which are
320 # different leaves and stay readable.
321 "token",
322 "api_token",
323 }
325 @staticmethod
326 def is_sensitive_setting(
327 key: str,
328 ui_element: str | None = None,
329 sensitive_keys: Set[str] | None = None,
330 ) -> bool:
331 """True when a setting holds a secret: it is ``ui_element ==
332 "password"`` OR the last dotted segment of its key names a secret
333 (``llm.openai.api_key`` -> ``api_key``), either exactly or as an
334 underscore-delimited suffix (``local_search_milvus_token`` ->
335 ``_token``). See ``_matches_sensitive_name`` for both arms and for
336 the qualifier carve-out that keeps ``requires_api_key`` readable.
338 Single source of truth for "is this a secret" so the GET redactor
339 and the write-back no-op guards apply the SAME predicate — a value
340 the redactor masks to the sentinel must also be one the guards
341 refuse to overwrite, or a redacted GET could round-trip the
342 sentinel back over the real secret.
343 """
344 if ui_element == "password":
345 return True
346 sens = {
347 k.lower()
348 for k in (sensitive_keys or DataSanitizer.DEFAULT_SENSITIVE_KEYS)
349 }
350 # _visible_leaf normalizes away invisible/whitespace padding so a key
351 # like "api_key " or "api_key<zero-width>" still matches (see its
352 # docstring). Keep the raw-leaf match as well: callers may provide a
353 # custom sensitive name containing one of those characters, and an
354 # exact match must remain sensitive for backward compatibility.
355 raw_leaf = key.rsplit(".", 1)[-1].lower()
356 return _matches_sensitive_name(
357 raw_leaf, sens
358 ) or _matches_sensitive_name(_visible_leaf(key), sens)
360 @staticmethod
361 def redact_value(
362 key: str,
363 ui_element: str | None = None,
364 value: Any = None,
365 sensitive_keys: Set[str] | None = None,
366 redaction_text: str = REDACTION_TEXT,
367 ) -> Any:
368 """Redact a single setting's value when it holds a set secret.
370 The single-value counterpart of ``redact_settings_snapshot``: it
371 applies the SAME ``is_sensitive_setting`` predicate and the SAME
372 empty-value rule, so every read path that ships a setting to the
373 browser (the bulk GET, the singular GET, the run-time snapshot)
374 masks identically. Returns ``redaction_text`` for a non-empty
375 sensitive value, otherwise ``value`` unchanged.
377 Empty values (``None``, ``""``, ``[]``, ``{}`` and whitespace-only
378 strings) are left readable so the UI can tell "configured" from "not
379 configured" without leaking that a secret is set.
381 Nested containers are redacted recursively: a subtree request such as
382 ``get_setting("llm")`` returns ``{"openai.api_key": "sk-…", …}`` where
383 the outer key (``llm``) is not sensitive but inner keys are, and a JSON
384 setting value may be a list of dicts (e.g. ``[{"api_key": "sk-…"}]``).
385 Without the recursion, ``GET /settings/api/bulk?keys[]=llm`` would ship
386 those nested secrets in the clear. A sensitive OUTER key still masks its
387 whole value wholesale (the check below runs first), so plain lists under
388 a non-sensitive key pass through untouched.
390 Non-string values are only redacted through the original exact-leaf
391 / ``password`` match (see ``_is_exact_sensitive_match``). The
392 broadened snake_case suffix match added for #5771 is a lexical
393 heuristic and must not fire on a bool/int/dict value: a checkbox or
394 number setting whose name happens to lexically match a sensitive
395 suffix cannot hold a real credential, and replacing its typed value
396 with the "[REDACTED]" string corrupts it in the settings UI (a
397 checkbox only renders ``checked`` for a literal ``True``) and can
398 silently write back the wrong value on the next save.
400 A dict/list value under a key that matches ONLY the broadened arm
401 (not the exact/password match) is neither a plain string nor caught
402 by the type guard above, so it falls through this first check
403 untouched. Left alone, the general recursion below would then mask
404 it by SUB-KEY name only -- missing a secret nested under a
405 non-sensitive sub-key such as ``{"value": "s3cr3t"}``. So that case
406 is handled separately: every string leaf inside the container is
407 force-masked (``_force_redact_strings``), while bool/int/None
408 leaves stay untouched, same as the type guard above.
409 """
410 if DataSanitizer.is_sensitive_setting(
411 key, ui_element, sensitive_keys
412 ) and not _is_empty_value(value):
413 sens = {
414 k.lower()
415 for k in (
416 sensitive_keys or DataSanitizer.DEFAULT_SENSITIVE_KEYS
417 )
418 }
419 if isinstance(value, str) or _is_exact_sensitive_match(
420 key, ui_element, sens
421 ):
422 return redaction_text
423 if isinstance(value, (dict, list)):
424 return _force_redact_strings(value, redaction_text)
425 if isinstance(value, dict):
426 redacted: dict = {}
427 for sub_key, sub_val in value.items():
428 nested_key = f"{key}.{sub_key}" if key else sub_key
429 redacted[sub_key] = DataSanitizer.redact_value(
430 nested_key, None, sub_val, sensitive_keys, redaction_text
431 )
432 return redacted
433 if isinstance(value, list):
434 # List items reuse the parent key; a dict item is caught by the
435 # branch above (its own sensitive leaves get masked), a plain
436 # scalar item passes through.
437 return [
438 DataSanitizer.redact_value(
439 key, None, item, sensitive_keys, redaction_text
440 )
441 for item in value
442 ]
443 return value
445 @staticmethod
446 def sanitize(data: Any, sensitive_keys: Set[str] | None = None) -> Any:
447 """
448 Recursively remove sensitive keys from data structures.
450 This method traverses dictionaries and lists, removing any keys that match
451 the sensitive keys list (case-insensitive). This prevents accidental
452 credential leakage in optimization results, logs, or API responses.
454 Args:
455 data: The data structure to sanitize (dict, list, or primitive)
456 sensitive_keys: Set of key names to remove (case-insensitive).
457 If None, uses DEFAULT_SENSITIVE_KEYS.
459 Returns:
460 Sanitized copy of the data with sensitive keys removed
462 Example:
463 >>> sanitizer = DataSanitizer()
464 >>> data = {"username": "user", "api_key": "secret123"}
465 >>> sanitizer.sanitize(data)
466 {"username": "user"}
467 """
468 if sensitive_keys is None:
469 sensitive_keys = DataSanitizer.DEFAULT_SENSITIVE_KEYS
471 # Convert to lowercase for case-insensitive comparison
472 sensitive_keys_lower = {key.lower() for key in sensitive_keys}
474 if isinstance(data, dict):
475 return {
476 k: DataSanitizer.sanitize(v, sensitive_keys)
477 for k, v in data.items()
478 if k.lower() not in sensitive_keys_lower
479 }
480 if isinstance(data, list):
481 return [
482 DataSanitizer.sanitize(item, sensitive_keys) for item in data
483 ]
484 # Return primitives unchanged
485 return data
487 @staticmethod
488 def redact(
489 data: Any,
490 sensitive_keys: Set[str] | None = None,
491 redaction_text: str = REDACTION_TEXT,
492 ) -> Any:
493 """
494 Recursively redact (replace with placeholder) sensitive values in data structures.
496 Unlike sanitize() which removes keys entirely, this method replaces their
497 values with a redaction placeholder, preserving the structure.
499 Args:
500 data: The data structure to redact (dict, list, or primitive)
501 sensitive_keys: Set of key names to redact (case-insensitive).
502 If None, uses DEFAULT_SENSITIVE_KEYS.
503 redaction_text: Text to replace sensitive values with
505 Returns:
506 Copy of the data with sensitive values redacted
508 Example:
509 >>> sanitizer = DataSanitizer()
510 >>> data = {"username": "user", "api_key": "secret123"}
511 >>> sanitizer.redact(data)
512 {"username": "user", "api_key": "[REDACTED]"}
513 """
514 if sensitive_keys is None:
515 sensitive_keys = DataSanitizer.DEFAULT_SENSITIVE_KEYS
517 # Convert to lowercase for case-insensitive comparison
518 sensitive_keys_lower = {key.lower() for key in sensitive_keys}
520 if isinstance(data, dict):
521 return {
522 k: (
523 redaction_text
524 if k.lower() in sensitive_keys_lower
525 else DataSanitizer.redact(v, sensitive_keys, redaction_text)
526 )
527 for k, v in data.items()
528 }
529 if isinstance(data, list):
530 return [
531 DataSanitizer.redact(item, sensitive_keys, redaction_text)
532 for item in data
533 ]
534 # Return primitives unchanged
535 return data
537 @staticmethod
538 def redact_settings_snapshot(
539 snapshot: Any,
540 sensitive_keys: Set[str] | None = None,
541 redaction_text: str = REDACTION_TEXT,
542 ) -> Any:
543 """Redact secret values in a settings snapshot while preserving metadata.
545 A settings snapshot from ``SettingsManager.get_all_settings()`` has the
546 nested-with-metadata shape ``{dotted_key: {"value": ..., "ui_element":
547 ..., "type": ..., ...}}``. The ordinary ``redact()`` method does not
548 catch secrets in this shape: the outer dotted key (e.g.
549 ``"llm.openai.api_key"``) is not in the sensitive-name set (only the
550 suffix ``"api_key"`` is), and the inner key ``"value"`` is not
551 sensitive — so the secret survives unredacted.
552 ``redact_settings_snapshot`` handles the shape correctly:
554 - Replaces ``entry["value"]`` with ``redaction_text`` when the entry
555 is sensitive (``ui_element == "password"`` OR the last dotted
556 segment of the outer key matches a sensitive name).
557 - Preserves all metadata (``ui_element``, ``type``, ``description``,
558 etc.) so YAML diffs can still show "this key existed."
559 - Leaves empty values (``None``, ``""``, ``[]``, ``{}`` and
560 whitespace-only strings) unredacted so diffs of "unset" settings
561 stay readable.
562 - Pure function: does not mutate the input.
564 Entries that don't have the metadata-wrapper shape (e.g. mixed
565 snapshots that contain bare values) are passed through untouched —
566 this is intentional so the helper is safe to call on any dict
567 without crashing.
569 Args:
570 snapshot: A settings snapshot dict.
571 sensitive_keys: Override the default set of sensitive name
572 suffixes. Defaults to ``DataSanitizer.DEFAULT_SENSITIVE_KEYS``.
573 redaction_text: Replacement string for redacted values.
575 Returns:
576 New dict with secret values replaced.
578 Example:
579 >>> snap = {"llm.openai.api_key": {"value": "sk-x", "ui_element": "password"}}
580 >>> DataSanitizer.redact_settings_snapshot(snap)
581 {'llm.openai.api_key': {'value': '[REDACTED]', 'ui_element': 'password'}}
582 """
583 if not isinstance(snapshot, dict):
584 return snapshot
586 out: dict = {}
587 for key, entry in snapshot.items():
588 if not isinstance(entry, dict) or "value" not in entry:
589 out[key] = entry
590 continue
591 new_entry = dict(entry) # shallow copy preserves metadata
592 # Delegate the per-value rule to redact_value so the snapshot,
593 # the singular GET and the bulk GET can never mask differently.
594 new_entry["value"] = DataSanitizer.redact_value(
595 key,
596 entry.get("ui_element"),
597 entry.get("value"),
598 sensitive_keys,
599 redaction_text,
600 )
601 out[key] = new_entry
602 return out
605# Convenience functions for direct use
606def sanitize_data(data: Any, sensitive_keys: Set[str] | None = None) -> Any:
607 """
608 Remove sensitive keys from data structures.
610 Convenience function that calls DataSanitizer.sanitize().
612 Args:
613 data: The data structure to sanitize
614 sensitive_keys: Optional set of sensitive key names
616 Returns:
617 Sanitized copy of the data
618 """
619 return DataSanitizer.sanitize(data, sensitive_keys)
622def redact_data(
623 data: Any,
624 sensitive_keys: Set[str] | None = None,
625 redaction_text: str = REDACTION_TEXT,
626) -> Any:
627 """
628 Redact (replace) sensitive values in data structures.
630 Convenience function that calls DataSanitizer.redact().
632 Args:
633 data: The data structure to redact
634 sensitive_keys: Optional set of sensitive key names
635 redaction_text: Text to replace sensitive values with
637 Returns:
638 Copy of the data with sensitive values redacted
639 """
640 return DataSanitizer.redact(data, sensitive_keys, redaction_text)
643def filter_research_metadata(research_meta: Any) -> dict:
644 """Filter research_meta to only safe fields for history list API responses.
646 Uses an allowlist approach to prevent leaking settings_snapshot
647 (which contains API keys, passwords, tokens) to the frontend.
648 History list consumers only need is_news_search from metadata.
650 Args:
651 research_meta: Raw research metadata (dict, JSON string, or None)
653 Returns:
654 dict with only safe fields extracted (currently: is_news_search)
655 """
656 try:
657 meta = research_meta or {}
658 if isinstance(meta, str):
659 meta = json.loads(meta)
660 if not isinstance(meta, dict):
661 return {"is_news_search": False}
662 return {
663 "is_news_search": bool(meta.get("is_news_search", False)),
664 }
665 except (json.JSONDecodeError, TypeError, AttributeError):
666 return {"is_news_search": False}
669def strip_settings_snapshot(research_meta: Any) -> dict:
670 """Remove private fields from research_meta for API responses.
672 settings_snapshot contains all application settings including API keys.
673 Legacy error strings can contain absolute report paths because older
674 writers persisted str(exception). Preserve the error field's shape for
675 clients, but allow only known authored messages through. Keep other
676 metadata fields that the frontend needs (phase, error_type,
677 processed_query, mode, duration, etc.).
679 Args:
680 research_meta: Raw research metadata (dict, JSON string, or None)
682 Returns:
683 Copy of the dict with settings_snapshot removed and error masked
684 """
685 try:
686 meta = research_meta or {}
687 if isinstance(meta, str):
688 meta = json.loads(meta)
689 if not isinstance(meta, dict):
690 return {}
691 public = {k: v for k, v in meta.items() if k != "settings_snapshot"}
692 if public.get("error"):
693 public["error"] = (
694 get_known_research_error_message(public["error"])
695 or PUBLIC_RESEARCH_ERROR
696 )
697 return public
698 except (json.JSONDecodeError, TypeError, AttributeError):
699 return {}