Coverage for src/local_deep_research/security/data_sanitizer.py: 99%
119 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""Security module for sanitizing sensitive data from data structures.
3This module ensures that sensitive information like API keys, passwords, and tokens
4are not accidentally leaked in logs, files, or API responses.
6Includes helpers for filtering research metadata in API responses to prevent
7settings_snapshot (which contains all application settings including API keys)
8from being sent to the frontend.
9"""
11import json
12from typing import Any, Set
15# The placeholder a redacted value is replaced with. Single source of truth
16# so that write-back guards (which must treat this sentinel as a no-op to
17# avoid persisting it over a real secret on a redacted GET round-trip)
18# cannot drift from what the redactor actually emits.
19REDACTION_TEXT = "[REDACTED]"
21# Unicode 16.0 DerivedCoreProperties.txt, Default_Ignorable_Code_Point.
22# Most Unicode Default_Ignorable_Code_Point characters are non-printable and
23# are removed by str.isprintable(). These exceptions are printable combining
24# marks or letter-category fillers, so list them explicitly. Do not drop Mn as
25# a category: ordinary combining marks are visible parts of legitimate names.
26_PRINTABLE_DEFAULT_IGNORABLES = (
27 frozenset({0x034F}) # COMBINING GRAPHEME JOINER
28 | frozenset(range(0x115F, 0x1161)) # Hangul fillers
29 | frozenset(range(0x17B4, 0x17B6)) # Khmer inherent vowels
30 | frozenset(range(0x180B, 0x1810)) # Mongolian variation separators
31 | frozenset({0x3164, 0xFFA0}) # Hangul fillers
32 | frozenset(range(0xFE00, 0xFE10)) # Variation Selectors
33 | frozenset(range(0xE0100, 0xE01F0)) # Variation Selectors Supplement
34)
36# Printable characters that Unicode specifies as rendering blank but that are
37# not default ignorables. They pass str.isprintable() and setting-key validation,
38# so they can otherwise disguise a sensitive leaf in the bulk settings endpoint.
39_PRINTABLE_BLANKS = frozenset(
40 {
41 0x2800, # BRAILLE PATTERN BLANK
42 0x13441, # EGYPTIAN HIEROGLYPH FULL BLANK
43 0x13442, # EGYPTIAN HIEROGLYPH HALF BLANK
44 0x1D159, # MUSICAL SYMBOL NULL NOTEHEAD
45 }
46)
49def _visible_leaf(key: str) -> str:
50 """The key's last dotted segment, normalized for the sensitive-name check.
52 Defense in depth for the bulk-settings GET, which lacks the
53 authoritative ``ui_element == "password"`` signal and falls back to the key
54 name. Drops non-printable chars (control, format \u2014 zero-width/BOM/bidi/tag
55 chars/soft hyphen \u2014 and non-ASCII spaces) plus printable Unicode default
56 ignorables (variation selectors, combining grapheme joiner, and fillers)
57 and explicitly known printable blank characters, then strips ASCII space
58 and lowercases. Ordinary visible combining marks are retained. This
59 intentionally fails closed: removing invisible padding can collapse a
60 distinct key onto a sensitive name and over-redact its value, which is
61 safer than exposing a potentially disguised secret.
62 """
63 leaf = key.rsplit(".", 1)[-1]
64 leaf = "".join(
65 ch
66 for ch in leaf
67 if ch.isprintable()
68 and ord(ch) not in _PRINTABLE_DEFAULT_IGNORABLES
69 and ord(ch) not in _PRINTABLE_BLANKS
70 )
71 return leaf.strip().lower()
74def _is_empty_value(value: Any) -> bool:
75 """True when a value counts as unconfigured for the redaction empty rule.
77 Covers ``None``, ``""``, ``[]``, ``{}`` plus whitespace-only strings:
78 the notification manager already treats ``" "`` as unconfigured
79 (``not service_urls.strip()``), so redacting it to the sentinel would
80 make an unset URL LOOK configured. Everything else (0, False,
81 non-empty strings, non-empty containers) counts as set.
82 """
83 if isinstance(value, str):
84 return not value.strip()
85 return value in (None, "", [], {})
88# Leading qualifiers that mark a leaf as a boolean FLAG *about* a secret rather
89# than the secret itself. ``search.engine.web.brave.requires_api_key`` is a
90# checkbox saying an engine needs a key; masking it would ship "[REDACTED]" to
91# the settings UI in place of true/false and make the write-back no-op guards
92# refuse legitimate writes. Consulted only by the underscore-suffix arm below —
93# a leaf that *equals* a sensitive name has no qualifier to strip.
94# This is a name-shape heuristic, so it is accepted as under-redaction risk
95# for a future setting that reuses one of these qualifier prefixes on a key
96# that actually stores a real string credential (rather than a boolean flag)
97# -- test_every_shipped_password_setting_is_redacted
98# (tests/security/test_bulk_secret_name_coverage.py) audits every shipped
99# password-typed setting's key shape and fails the moment such a setting
100# ships, so the gap does not survive silently.
101_NON_SECRET_LEAF_PREFIXES = (
102 "allow_",
103 "allows_",
104 "disable_",
105 "enable_",
106 "enabled_",
107 "has_",
108 "is_",
109 "need_",
110 "needs_",
111 "require_",
112 "requires_",
113 "show_",
114 "skip_",
115 "support_",
116 "supports_",
117 "use_",
118 "uses_",
119)
122def _has_non_secret_qualifier(leaf: str) -> bool:
123 """True when ``leaf`` carries a ``_NON_SECRET_LEAF_PREFIXES`` qualifier
124 at ANY underscore segment boundary, not only at the very start of the
125 string.
127 A dotted key's leaf is a single segment (``requires_api_key``), so a
128 start-of-string check is enough on its own. But a flat snake_case key
129 has no dots at all, so its "leaf" (see ``_matches_sensitive_name``) is
130 the ENTIRE key — a qualifier that isn't the first segment, e.g.
131 ``llm_requires_api_key``, would slip past a ``startswith`` check even
132 though it is the exact same "flag about a secret" shape as
133 ``requires_api_key``. Checking every underscore boundary (the start of
134 the leaf, or immediately after an ``_``) catches the qualifier wherever
135 it falls in the key, at the cost of also matching it as a middle
136 segment (e.g. ``foo_use_bar``) — intentional, since the prefix list is
137 a name-shape heuristic, not a position rule.
138 """
139 return any(
140 leaf.startswith(prefix) or f"_{prefix}" in leaf
141 for prefix in _NON_SECRET_LEAF_PREFIXES
142 )
145def _matches_sensitive_name(leaf: str, sensitive_names: Set[str]) -> bool:
146 """True when a normalized leaf names a secret.
148 Two arms:
150 1. Exact match (``llm.openai.api_key`` -> ``api_key``).
151 2. Underscore-boundary suffix match. Settings keys use two separator
152 conventions: dotted (``llm.openai.api_key``) and flat snake_case
153 (``local_search_milvus_token``). A flat key has no dots, so its dotted
154 "leaf" is the entire key and arm 1 can never match it — which is how a
155 password-typed setting shipped in the clear from the bulk settings GET
156 (#5762). Note the miss was never specific to ``token``:
157 ``local_search_milvus_api_key`` or ``some_password`` would have evaded
158 arm 1 just as completely.
160 Arm 2 skips leaves carrying a ``_NON_SECRET_LEAF_PREFIXES`` qualifier
161 anywhere at an underscore boundary (see ``_has_non_secret_qualifier``),
162 so both ``requires_api_key`` (dotted leaf) and ``llm_requires_api_key``
163 (flat leaf, qualifier not in the first segment) stay readable while
164 ``milvus_api_key`` does not. Plural token counts (``max_tokens``,
165 ``supports_max_tokens``) end in ``_tokens``, a different suffix than
166 ``_token``, and stay readable too.
167 """
168 if leaf in sensitive_names:
169 return True
170 if _has_non_secret_qualifier(leaf):
171 return False
172 return any(leaf.endswith(f"_{name}") for name in sensitive_names)
175def _is_exact_sensitive_match(
176 key: str, ui_element: str | None, sensitive_names: Set[str]
177) -> bool:
178 """True under the ORIGINAL (pre-#5771) sensitivity rule: ``ui_element ==
179 "password"`` or an exact dotted-leaf match — i.e. everything
180 ``is_sensitive_setting`` matched before the broadened underscore-suffix
181 arm (arm 2 of ``_matches_sensitive_name``) was added.
183 ``redact_value`` uses this to scope its non-string type guard to only
184 the NEW broadened match. The exact/password arms have masked non-string
185 values (e.g. a boolean under a key literally named ``api_key``) since
186 before this PR; that established behavior is intentionally left as-is.
187 The broadened arm is a lexical suffix heuristic backed by a
188 hand-maintained non-secret-prefix carve-out (``_NON_SECRET_LEAF_PREFIXES``),
189 so a boolean/number setting whose name happens to lexically match a
190 sensitive suffix (a future ``verify_token``-style flag the carve-out
191 list doesn't yet know about) must NOT be redacted: replacing a bool/int
192 with the "[REDACTED]" string corrupts it in the settings UI, which
193 only special-cases real booleans when rendering a checkbox
194 (``web/static/js/components/settings.js``), and silently writes back
195 ``false``/empty on the next save.
196 """
197 if ui_element == "password": 197 ↛ 198line 197 didn't jump to line 198 because the condition on line 197 was never true
198 return True
199 raw_leaf = key.rsplit(".", 1)[-1].lower()
200 return raw_leaf in sensitive_names or _visible_leaf(key) in sensitive_names
203def _force_redact_strings(value: Any, redaction_text: str) -> Any:
204 """Mask every string leaf inside a container, regardless of its own
205 sub-key name, while leaving bool/int/None leaves untouched.
207 Used when a KEY matches the broadened suffix-only sensitive arm (arm 2
208 of ``_matches_sensitive_name``) and its VALUE is a dict/list rather than
209 a plain string. ``redact_value``'s normal recursion masks by SUB-KEY
210 name, so a secret nested under a non-sensitive sub-key -- e.g.
211 ``{"value": "s3cr3t"}`` under a ``milvus_token`` setting -- would
212 otherwise survive unredacted: ``value`` is not itself a sensitive leaf
213 name. This walks the container and replaces every non-empty string leaf
214 with the sentinel unconditionally, so the secret cannot hide behind an
215 innocuous sub-key. It preserves round 2's guarantee that a bool/int
216 leaf is never corrupted into a string, and leaves empty/whitespace-only
217 string leaves readable so an unset nested field doesn't look configured
218 (matching ``_is_empty_value``'s rule for the top-level value).
219 """
220 if isinstance(value, str):
221 return redaction_text if value.strip() else value
222 if isinstance(value, dict):
223 return {
224 sub_key: _force_redact_strings(sub_val, redaction_text)
225 for sub_key, sub_val in value.items()
226 }
227 if isinstance(value, list):
228 return [_force_redact_strings(item, redaction_text) for item in value]
229 return value
232class DataSanitizer:
233 """Utility class for removing sensitive information from data structures."""
235 # Public alias of the module-level sentinel (see REDACTION_TEXT above).
236 REDACTION_TEXT: str = REDACTION_TEXT
238 # Default set of sensitive key names to redact
239 DEFAULT_SENSITIVE_KEYS: Set[str] = {
240 "api_key",
241 "apikey",
242 "password",
243 "secret",
244 "access_token",
245 "refresh_token",
246 "private_key",
247 "auth_token",
248 "session_token",
249 "csrf_token",
250 # Additional unambiguous secret leaf-names. The predicate is
251 # exact-match on the last dotted segment, so these matter especially
252 # for the bulk settings GET, which can only use the key-name heuristic
253 # (it passes no ui_element). All are unambiguously secrets and no
254 # current setting key uses them for non-secret data.
255 "client_secret",
256 "secret_key",
257 "bearer_token",
258 "api_secret",
259 "app_secret",
260 # The notification service URL (apprise-style) embeds credentials --
261 # e.g. mailto://user:pass@host, discord://webhook_id/token, Slack/ntfy
262 # tokens. The app already masks it in logs (mask_sensitive_url); this
263 # keeps it out of the settings API read paths too. Only
264 # notifications.service_url has this leaf, so no other setting is
265 # affected. The empty-value rule keeps an unconfigured URL readable.
266 "service_url",
267 # A bare "token" leaf. The qualified spellings above (access_token,
268 # auth_token, bearer_token, ...) left the unqualified name uncovered,
269 # so a setting keyed "<prefix>_token" with no dots in it (its leaf is
270 # the whole key) matched nothing and shipped in the clear from the
271 # bulk settings GET (#5762). "token" and "api_token" name a secret in
272 # every credential convention we ship; token COUNTS are spelled
273 # max_tokens / context_tokens / supports_max_tokens, which are
274 # different leaves and stay readable.
275 "token",
276 "api_token",
277 }
279 @staticmethod
280 def is_sensitive_setting(
281 key: str,
282 ui_element: str | None = None,
283 sensitive_keys: Set[str] | None = None,
284 ) -> bool:
285 """True when a setting holds a secret: it is ``ui_element ==
286 "password"`` OR the last dotted segment of its key names a secret
287 (``llm.openai.api_key`` -> ``api_key``), either exactly or as an
288 underscore-delimited suffix (``local_search_milvus_token`` ->
289 ``_token``). See ``_matches_sensitive_name`` for both arms and for
290 the qualifier carve-out that keeps ``requires_api_key`` readable.
292 Single source of truth for "is this a secret" so the GET redactor
293 and the write-back no-op guards apply the SAME predicate — a value
294 the redactor masks to the sentinel must also be one the guards
295 refuse to overwrite, or a redacted GET could round-trip the
296 sentinel back over the real secret.
297 """
298 if ui_element == "password":
299 return True
300 sens = {
301 k.lower()
302 for k in (sensitive_keys or DataSanitizer.DEFAULT_SENSITIVE_KEYS)
303 }
304 # _visible_leaf normalizes away invisible/whitespace padding so a key
305 # like "api_key " or "api_key<zero-width>" still matches (see its
306 # docstring). Keep the raw-leaf match as well: callers may provide a
307 # custom sensitive name containing one of those characters, and an
308 # exact match must remain sensitive for backward compatibility.
309 raw_leaf = key.rsplit(".", 1)[-1].lower()
310 return _matches_sensitive_name(
311 raw_leaf, sens
312 ) or _matches_sensitive_name(_visible_leaf(key), sens)
314 @staticmethod
315 def redact_value(
316 key: str,
317 ui_element: str | None = None,
318 value: Any = None,
319 sensitive_keys: Set[str] | None = None,
320 redaction_text: str = REDACTION_TEXT,
321 ) -> Any:
322 """Redact a single setting's value when it holds a set secret.
324 The single-value counterpart of ``redact_settings_snapshot``: it
325 applies the SAME ``is_sensitive_setting`` predicate and the SAME
326 empty-value rule, so every read path that ships a setting to the
327 browser (the bulk GET, the singular GET, the run-time snapshot)
328 masks identically. Returns ``redaction_text`` for a non-empty
329 sensitive value, otherwise ``value`` unchanged.
331 Empty values (``None``, ``""``, ``[]``, ``{}`` and whitespace-only
332 strings) are left readable so the UI can tell "configured" from "not
333 configured" without leaking that a secret is set.
335 Nested containers are redacted recursively: a subtree request such as
336 ``get_setting("llm")`` returns ``{"openai.api_key": "sk-…", …}`` where
337 the outer key (``llm``) is not sensitive but inner keys are, and a JSON
338 setting value may be a list of dicts (e.g. ``[{"api_key": "sk-…"}]``).
339 Without the recursion, ``GET /settings/api/bulk?keys[]=llm`` would ship
340 those nested secrets in the clear. A sensitive OUTER key still masks its
341 whole value wholesale (the check below runs first), so plain lists under
342 a non-sensitive key pass through untouched.
344 Non-string values are only redacted through the original exact-leaf
345 / ``password`` match (see ``_is_exact_sensitive_match``). The
346 broadened snake_case suffix match added for #5771 is a lexical
347 heuristic and must not fire on a bool/int/dict value: a checkbox or
348 number setting whose name happens to lexically match a sensitive
349 suffix cannot hold a real credential, and replacing its typed value
350 with the "[REDACTED]" string corrupts it in the settings UI (a
351 checkbox only renders ``checked`` for a literal ``True``) and can
352 silently write back the wrong value on the next save.
354 A dict/list value under a key that matches ONLY the broadened arm
355 (not the exact/password match) is neither a plain string nor caught
356 by the type guard above, so it falls through this first check
357 untouched. Left alone, the general recursion below would then mask
358 it by SUB-KEY name only -- missing a secret nested under a
359 non-sensitive sub-key such as ``{"value": "s3cr3t"}``. So that case
360 is handled separately: every string leaf inside the container is
361 force-masked (``_force_redact_strings``), while bool/int/None
362 leaves stay untouched, same as the type guard above.
363 """
364 if DataSanitizer.is_sensitive_setting(
365 key, ui_element, sensitive_keys
366 ) and not _is_empty_value(value):
367 sens = {
368 k.lower()
369 for k in (
370 sensitive_keys or DataSanitizer.DEFAULT_SENSITIVE_KEYS
371 )
372 }
373 if isinstance(value, str) or _is_exact_sensitive_match(
374 key, ui_element, sens
375 ):
376 return redaction_text
377 if isinstance(value, (dict, list)):
378 return _force_redact_strings(value, redaction_text)
379 if isinstance(value, dict):
380 redacted: dict = {}
381 for sub_key, sub_val in value.items():
382 nested_key = f"{key}.{sub_key}" if key else sub_key
383 redacted[sub_key] = DataSanitizer.redact_value(
384 nested_key, None, sub_val, sensitive_keys, redaction_text
385 )
386 return redacted
387 if isinstance(value, list):
388 # List items reuse the parent key; a dict item is caught by the
389 # branch above (its own sensitive leaves get masked), a plain
390 # scalar item passes through.
391 return [
392 DataSanitizer.redact_value(
393 key, None, item, sensitive_keys, redaction_text
394 )
395 for item in value
396 ]
397 return value
399 @staticmethod
400 def sanitize(data: Any, sensitive_keys: Set[str] | None = None) -> Any:
401 """
402 Recursively remove sensitive keys from data structures.
404 This method traverses dictionaries and lists, removing any keys that match
405 the sensitive keys list (case-insensitive). This prevents accidental
406 credential leakage in optimization results, logs, or API responses.
408 Args:
409 data: The data structure to sanitize (dict, list, or primitive)
410 sensitive_keys: Set of key names to remove (case-insensitive).
411 If None, uses DEFAULT_SENSITIVE_KEYS.
413 Returns:
414 Sanitized copy of the data with sensitive keys removed
416 Example:
417 >>> sanitizer = DataSanitizer()
418 >>> data = {"username": "user", "api_key": "secret123"}
419 >>> sanitizer.sanitize(data)
420 {"username": "user"}
421 """
422 if sensitive_keys is None:
423 sensitive_keys = DataSanitizer.DEFAULT_SENSITIVE_KEYS
425 # Convert to lowercase for case-insensitive comparison
426 sensitive_keys_lower = {key.lower() for key in sensitive_keys}
428 if isinstance(data, dict):
429 return {
430 k: DataSanitizer.sanitize(v, sensitive_keys)
431 for k, v in data.items()
432 if k.lower() not in sensitive_keys_lower
433 }
434 if isinstance(data, list):
435 return [
436 DataSanitizer.sanitize(item, sensitive_keys) for item in data
437 ]
438 # Return primitives unchanged
439 return data
441 @staticmethod
442 def redact(
443 data: Any,
444 sensitive_keys: Set[str] | None = None,
445 redaction_text: str = REDACTION_TEXT,
446 ) -> Any:
447 """
448 Recursively redact (replace with placeholder) sensitive values in data structures.
450 Unlike sanitize() which removes keys entirely, this method replaces their
451 values with a redaction placeholder, preserving the structure.
453 Args:
454 data: The data structure to redact (dict, list, or primitive)
455 sensitive_keys: Set of key names to redact (case-insensitive).
456 If None, uses DEFAULT_SENSITIVE_KEYS.
457 redaction_text: Text to replace sensitive values with
459 Returns:
460 Copy of the data with sensitive values redacted
462 Example:
463 >>> sanitizer = DataSanitizer()
464 >>> data = {"username": "user", "api_key": "secret123"}
465 >>> sanitizer.redact(data)
466 {"username": "user", "api_key": "[REDACTED]"}
467 """
468 if sensitive_keys is None:
469 sensitive_keys = DataSanitizer.DEFAULT_SENSITIVE_KEYS
471 # Convert to lowercase for case-insensitive comparison
472 sensitive_keys_lower = {key.lower() for key in sensitive_keys}
474 if isinstance(data, dict):
475 return {
476 k: (
477 redaction_text
478 if k.lower() in sensitive_keys_lower
479 else DataSanitizer.redact(v, sensitive_keys, redaction_text)
480 )
481 for k, v in data.items()
482 }
483 if isinstance(data, list):
484 return [
485 DataSanitizer.redact(item, sensitive_keys, redaction_text)
486 for item in data
487 ]
488 # Return primitives unchanged
489 return data
491 @staticmethod
492 def redact_settings_snapshot(
493 snapshot: Any,
494 sensitive_keys: Set[str] | None = None,
495 redaction_text: str = REDACTION_TEXT,
496 ) -> Any:
497 """Redact secret values in a settings snapshot while preserving metadata.
499 A settings snapshot from ``SettingsManager.get_all_settings()`` has the
500 nested-with-metadata shape ``{dotted_key: {"value": ..., "ui_element":
501 ..., "type": ..., ...}}``. The ordinary ``redact()`` method does not
502 catch secrets in this shape: the outer dotted key (e.g.
503 ``"llm.openai.api_key"``) is not in the sensitive-name set (only the
504 suffix ``"api_key"`` is), and the inner key ``"value"`` is not
505 sensitive — so the secret survives unredacted.
506 ``redact_settings_snapshot`` handles the shape correctly:
508 - Replaces ``entry["value"]`` with ``redaction_text`` when the entry
509 is sensitive (``ui_element == "password"`` OR the last dotted
510 segment of the outer key matches a sensitive name).
511 - Preserves all metadata (``ui_element``, ``type``, ``description``,
512 etc.) so YAML diffs can still show "this key existed."
513 - Leaves empty values (``None``, ``""``, ``[]``, ``{}`` and
514 whitespace-only strings) unredacted so diffs of "unset" settings
515 stay readable.
516 - Pure function: does not mutate the input.
518 Entries that don't have the metadata-wrapper shape (e.g. mixed
519 snapshots that contain bare values) are passed through untouched —
520 this is intentional so the helper is safe to call on any dict
521 without crashing.
523 Args:
524 snapshot: A settings snapshot dict.
525 sensitive_keys: Override the default set of sensitive name
526 suffixes. Defaults to ``DataSanitizer.DEFAULT_SENSITIVE_KEYS``.
527 redaction_text: Replacement string for redacted values.
529 Returns:
530 New dict with secret values replaced.
532 Example:
533 >>> snap = {"llm.openai.api_key": {"value": "sk-x", "ui_element": "password"}}
534 >>> DataSanitizer.redact_settings_snapshot(snap)
535 {'llm.openai.api_key': {'value': '[REDACTED]', 'ui_element': 'password'}}
536 """
537 if not isinstance(snapshot, dict):
538 return snapshot
540 out: dict = {}
541 for key, entry in snapshot.items():
542 if not isinstance(entry, dict) or "value" not in entry:
543 out[key] = entry
544 continue
545 new_entry = dict(entry) # shallow copy preserves metadata
546 # Delegate the per-value rule to redact_value so the snapshot,
547 # the singular GET and the bulk GET can never mask differently.
548 new_entry["value"] = DataSanitizer.redact_value(
549 key,
550 entry.get("ui_element"),
551 entry.get("value"),
552 sensitive_keys,
553 redaction_text,
554 )
555 out[key] = new_entry
556 return out
559# Convenience functions for direct use
560def sanitize_data(data: Any, sensitive_keys: Set[str] | None = None) -> Any:
561 """
562 Remove sensitive keys from data structures.
564 Convenience function that calls DataSanitizer.sanitize().
566 Args:
567 data: The data structure to sanitize
568 sensitive_keys: Optional set of sensitive key names
570 Returns:
571 Sanitized copy of the data
572 """
573 return DataSanitizer.sanitize(data, sensitive_keys)
576def redact_data(
577 data: Any,
578 sensitive_keys: Set[str] | None = None,
579 redaction_text: str = REDACTION_TEXT,
580) -> Any:
581 """
582 Redact (replace) sensitive values in data structures.
584 Convenience function that calls DataSanitizer.redact().
586 Args:
587 data: The data structure to redact
588 sensitive_keys: Optional set of sensitive key names
589 redaction_text: Text to replace sensitive values with
591 Returns:
592 Copy of the data with sensitive values redacted
593 """
594 return DataSanitizer.redact(data, sensitive_keys, redaction_text)
597def filter_research_metadata(research_meta: Any) -> dict:
598 """Filter research_meta to only safe fields for history list API responses.
600 Uses an allowlist approach to prevent leaking settings_snapshot
601 (which contains API keys, passwords, tokens) to the frontend.
602 History list consumers only need is_news_search from metadata.
604 Args:
605 research_meta: Raw research metadata (dict, JSON string, or None)
607 Returns:
608 dict with only safe fields extracted (currently: is_news_search)
609 """
610 try:
611 meta = research_meta or {}
612 if isinstance(meta, str):
613 meta = json.loads(meta)
614 if not isinstance(meta, dict):
615 return {"is_news_search": False}
616 return {
617 "is_news_search": bool(meta.get("is_news_search", False)),
618 }
619 except (json.JSONDecodeError, TypeError, AttributeError):
620 return {"is_news_search": False}
623def strip_settings_snapshot(research_meta: Any) -> dict:
624 """Remove settings_snapshot from research_meta for API responses.
626 settings_snapshot contains all application settings including API keys.
627 This strips it while preserving all other metadata fields that the
628 frontend needs (phase, error_type, processed_query, mode, duration, etc.).
630 Args:
631 research_meta: Raw research metadata (dict, JSON string, or None)
633 Returns:
634 Copy of the dict with settings_snapshot removed
635 """
636 try:
637 meta = research_meta or {}
638 if isinstance(meta, str):
639 meta = json.loads(meta)
640 if not isinstance(meta, dict):
641 return {}
642 return {k: v for k, v in meta.items() if k != "settings_snapshot"}
643 except (json.JSONDecodeError, TypeError, AttributeError):
644 return {}