Coverage for src/local_deep_research/security/data_sanitizer.py: 100%
89 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-20 01:24 +0000
1"""Security module for sanitizing sensitive data from data structures.
3This module ensures that sensitive information like API keys, passwords, and tokens
4are not accidentally leaked in logs, files, or API responses.
6Includes helpers for filtering research metadata in API responses to prevent
7settings_snapshot (which contains all application settings including API keys)
8from being sent to the frontend.
9"""
11import json
12from typing import Any, Set
15# The placeholder a redacted value is replaced with. Single source of truth
16# so that write-back guards (which must treat this sentinel as a no-op to
17# avoid persisting it over a real secret on a redacted GET round-trip)
18# cannot drift from what the redactor actually emits.
19REDACTION_TEXT = "[REDACTED]"
21# Unicode 16.0 DerivedCoreProperties.txt, Default_Ignorable_Code_Point.
22# Most Unicode Default_Ignorable_Code_Point characters are non-printable and
23# are removed by str.isprintable(). These exceptions are printable combining
24# marks or letter-category fillers, so list them explicitly. Do not drop Mn as
25# a category: ordinary combining marks are visible parts of legitimate names.
26_PRINTABLE_DEFAULT_IGNORABLES = (
27 frozenset({0x034F}) # COMBINING GRAPHEME JOINER
28 | frozenset(range(0x115F, 0x1161)) # Hangul fillers
29 | frozenset(range(0x17B4, 0x17B6)) # Khmer inherent vowels
30 | frozenset(range(0x180B, 0x1810)) # Mongolian variation separators
31 | frozenset({0x3164, 0xFFA0}) # Hangul fillers
32 | frozenset(range(0xFE00, 0xFE10)) # Variation Selectors
33 | frozenset(range(0xE0100, 0xE01F0)) # Variation Selectors Supplement
34)
36# Printable characters that Unicode specifies as rendering blank but that are
37# not default ignorables. They pass str.isprintable() and setting-key validation,
38# so they can otherwise disguise a sensitive leaf in the bulk settings endpoint.
39_PRINTABLE_BLANKS = frozenset(
40 {
41 0x2800, # BRAILLE PATTERN BLANK
42 0x13441, # EGYPTIAN HIEROGLYPH FULL BLANK
43 0x13442, # EGYPTIAN HIEROGLYPH HALF BLANK
44 0x1D159, # MUSICAL SYMBOL NULL NOTEHEAD
45 }
46)
49def _visible_leaf(key: str) -> str:
50 """The key's last dotted segment, normalized for the sensitive-name check.
52 Defense in depth for the bulk-settings GET, which lacks the
53 authoritative ``ui_element == "password"`` signal and falls back to the key
54 name. Drops non-printable chars (control, format \u2014 zero-width/BOM/bidi/tag
55 chars/soft hyphen \u2014 and non-ASCII spaces) plus printable Unicode default
56 ignorables (variation selectors, combining grapheme joiner, and fillers)
57 and explicitly known printable blank characters, then strips ASCII space
58 and lowercases. Ordinary visible combining marks are retained. This
59 intentionally fails closed: removing invisible padding can collapse a
60 distinct key onto a sensitive name and over-redact its value, which is
61 safer than exposing a potentially disguised secret.
62 """
63 leaf = key.rsplit(".", 1)[-1]
64 leaf = "".join(
65 ch
66 for ch in leaf
67 if ch.isprintable()
68 and ord(ch) not in _PRINTABLE_DEFAULT_IGNORABLES
69 and ord(ch) not in _PRINTABLE_BLANKS
70 )
71 return leaf.strip().lower()
74class DataSanitizer:
75 """Utility class for removing sensitive information from data structures."""
77 # Public alias of the module-level sentinel (see REDACTION_TEXT above).
78 REDACTION_TEXT: str = REDACTION_TEXT
80 # Default set of sensitive key names to redact
81 DEFAULT_SENSITIVE_KEYS: Set[str] = {
82 "api_key",
83 "apikey",
84 "password",
85 "secret",
86 "access_token",
87 "refresh_token",
88 "private_key",
89 "auth_token",
90 "session_token",
91 "csrf_token",
92 # Additional unambiguous secret leaf-names. The predicate is
93 # exact-match on the last dotted segment, so these matter especially
94 # for the bulk settings GET, which can only use the key-name heuristic
95 # (it passes no ui_element). All are unambiguously secrets and no
96 # current setting key uses them for non-secret data.
97 "client_secret",
98 "secret_key",
99 "bearer_token",
100 "api_secret",
101 "app_secret",
102 }
104 @staticmethod
105 def is_sensitive_setting(
106 key: str,
107 ui_element: str | None = None,
108 sensitive_keys: Set[str] | None = None,
109 ) -> bool:
110 """True when a setting holds a secret: it is ``ui_element ==
111 "password"`` OR the last dotted segment of its key is a sensitive
112 name (``llm.openai.api_key`` -> ``api_key``).
114 Single source of truth for "is this a secret" so the GET redactor
115 and the write-back no-op guards apply the SAME predicate — a value
116 the redactor masks to the sentinel must also be one the guards
117 refuse to overwrite, or a redacted GET could round-trip the
118 sentinel back over the real secret.
119 """
120 if ui_element == "password":
121 return True
122 sens = {
123 k.lower()
124 for k in (sensitive_keys or DataSanitizer.DEFAULT_SENSITIVE_KEYS)
125 }
126 # _visible_leaf normalizes away invisible/whitespace padding so a key
127 # like "api_key " or "api_key<zero-width>" still matches (see its
128 # docstring). Keep the raw-leaf match as well: callers may provide a
129 # custom sensitive name containing one of those characters, and an
130 # exact match must remain sensitive for backward compatibility.
131 raw_leaf = key.rsplit(".", 1)[-1].lower()
132 return raw_leaf in sens or _visible_leaf(key) in sens
134 @staticmethod
135 def redact_value(
136 key: str,
137 ui_element: str | None = None,
138 value: Any = None,
139 sensitive_keys: Set[str] | None = None,
140 redaction_text: str = REDACTION_TEXT,
141 ) -> Any:
142 """Redact a single setting's value when it holds a set secret.
144 The single-value counterpart of ``redact_settings_snapshot``: it
145 applies the SAME ``is_sensitive_setting`` predicate and the SAME
146 empty-value rule, so every read path that ships a setting to the
147 browser (the bulk GET, the singular GET, the run-time snapshot)
148 masks identically. Returns ``redaction_text`` for a non-empty
149 sensitive value, otherwise ``value`` unchanged.
151 Empty values (``None``, ``""``, ``[]``, ``{}``) are left readable so
152 the UI can tell "configured" from "not configured" without leaking
153 that a secret is set.
155 Nested containers are redacted recursively: a subtree request such as
156 ``get_setting("llm")`` returns ``{"openai.api_key": "sk-…", …}`` where
157 the outer key (``llm``) is not sensitive but inner keys are, and a JSON
158 setting value may be a list of dicts (e.g. ``[{"api_key": "sk-…"}]``).
159 Without the recursion, ``GET /settings/api/bulk?keys[]=llm`` would ship
160 those nested secrets in the clear. A sensitive OUTER key still masks its
161 whole value wholesale (the check below runs first), so plain lists under
162 a non-sensitive key pass through untouched.
163 """
164 if DataSanitizer.is_sensitive_setting(
165 key, ui_element, sensitive_keys
166 ) and value not in (None, "", [], {}):
167 return redaction_text
168 if isinstance(value, dict):
169 redacted: dict = {}
170 for sub_key, sub_val in value.items():
171 nested_key = f"{key}.{sub_key}" if key else sub_key
172 redacted[sub_key] = DataSanitizer.redact_value(
173 nested_key, None, sub_val, sensitive_keys, redaction_text
174 )
175 return redacted
176 if isinstance(value, list):
177 # List items reuse the parent key; a dict item is caught by the
178 # branch above (its own sensitive leaves get masked), a plain
179 # scalar item passes through.
180 return [
181 DataSanitizer.redact_value(
182 key, None, item, sensitive_keys, redaction_text
183 )
184 for item in value
185 ]
186 return value
188 @staticmethod
189 def sanitize(data: Any, sensitive_keys: Set[str] | None = None) -> Any:
190 """
191 Recursively remove sensitive keys from data structures.
193 This method traverses dictionaries and lists, removing any keys that match
194 the sensitive keys list (case-insensitive). This prevents accidental
195 credential leakage in optimization results, logs, or API responses.
197 Args:
198 data: The data structure to sanitize (dict, list, or primitive)
199 sensitive_keys: Set of key names to remove (case-insensitive).
200 If None, uses DEFAULT_SENSITIVE_KEYS.
202 Returns:
203 Sanitized copy of the data with sensitive keys removed
205 Example:
206 >>> sanitizer = DataSanitizer()
207 >>> data = {"username": "user", "api_key": "secret123"}
208 >>> sanitizer.sanitize(data)
209 {"username": "user"}
210 """
211 if sensitive_keys is None:
212 sensitive_keys = DataSanitizer.DEFAULT_SENSITIVE_KEYS
214 # Convert to lowercase for case-insensitive comparison
215 sensitive_keys_lower = {key.lower() for key in sensitive_keys}
217 if isinstance(data, dict):
218 return {
219 k: DataSanitizer.sanitize(v, sensitive_keys)
220 for k, v in data.items()
221 if k.lower() not in sensitive_keys_lower
222 }
223 if isinstance(data, list):
224 return [
225 DataSanitizer.sanitize(item, sensitive_keys) for item in data
226 ]
227 # Return primitives unchanged
228 return data
230 @staticmethod
231 def redact(
232 data: Any,
233 sensitive_keys: Set[str] | None = None,
234 redaction_text: str = REDACTION_TEXT,
235 ) -> Any:
236 """
237 Recursively redact (replace with placeholder) sensitive values in data structures.
239 Unlike sanitize() which removes keys entirely, this method replaces their
240 values with a redaction placeholder, preserving the structure.
242 Args:
243 data: The data structure to redact (dict, list, or primitive)
244 sensitive_keys: Set of key names to redact (case-insensitive).
245 If None, uses DEFAULT_SENSITIVE_KEYS.
246 redaction_text: Text to replace sensitive values with
248 Returns:
249 Copy of the data with sensitive values redacted
251 Example:
252 >>> sanitizer = DataSanitizer()
253 >>> data = {"username": "user", "api_key": "secret123"}
254 >>> sanitizer.redact(data)
255 {"username": "user", "api_key": "[REDACTED]"}
256 """
257 if sensitive_keys is None:
258 sensitive_keys = DataSanitizer.DEFAULT_SENSITIVE_KEYS
260 # Convert to lowercase for case-insensitive comparison
261 sensitive_keys_lower = {key.lower() for key in sensitive_keys}
263 if isinstance(data, dict):
264 return {
265 k: (
266 redaction_text
267 if k.lower() in sensitive_keys_lower
268 else DataSanitizer.redact(v, sensitive_keys, redaction_text)
269 )
270 for k, v in data.items()
271 }
272 if isinstance(data, list):
273 return [
274 DataSanitizer.redact(item, sensitive_keys, redaction_text)
275 for item in data
276 ]
277 # Return primitives unchanged
278 return data
280 @staticmethod
281 def redact_settings_snapshot(
282 snapshot: Any,
283 sensitive_keys: Set[str] | None = None,
284 redaction_text: str = REDACTION_TEXT,
285 ) -> Any:
286 """Redact secret values in a settings snapshot while preserving metadata.
288 A settings snapshot from ``SettingsManager.get_all_settings()`` has the
289 nested-with-metadata shape ``{dotted_key: {"value": ..., "ui_element":
290 ..., "type": ..., ...}}``. The ordinary ``redact()`` method does not
291 catch secrets in this shape: the outer dotted key (e.g.
292 ``"llm.openai.api_key"``) is not in the sensitive-name set (only the
293 suffix ``"api_key"`` is), and the inner key ``"value"`` is not
294 sensitive — so the secret survives unredacted.
295 ``redact_settings_snapshot`` handles the shape correctly:
297 - Replaces ``entry["value"]`` with ``redaction_text`` when the entry
298 is sensitive (``ui_element == "password"`` OR the last dotted
299 segment of the outer key matches a sensitive name).
300 - Preserves all metadata (``ui_element``, ``type``, ``description``,
301 etc.) so YAML diffs can still show "this key existed."
302 - Leaves empty values (``None``, ``""``, ``[]``, ``{}``) unredacted
303 so diffs of "unset" settings stay readable.
304 - Pure function: does not mutate the input.
306 Entries that don't have the metadata-wrapper shape (e.g. mixed
307 snapshots that contain bare values) are passed through untouched —
308 this is intentional so the helper is safe to call on any dict
309 without crashing.
311 Args:
312 snapshot: A settings snapshot dict.
313 sensitive_keys: Override the default set of sensitive name
314 suffixes. Defaults to ``DataSanitizer.DEFAULT_SENSITIVE_KEYS``.
315 redaction_text: Replacement string for redacted values.
317 Returns:
318 New dict with secret values replaced.
320 Example:
321 >>> snap = {"llm.openai.api_key": {"value": "sk-x", "ui_element": "password"}}
322 >>> DataSanitizer.redact_settings_snapshot(snap)
323 {'llm.openai.api_key': {'value': '[REDACTED]', 'ui_element': 'password'}}
324 """
325 if not isinstance(snapshot, dict):
326 return snapshot
328 out: dict = {}
329 for key, entry in snapshot.items():
330 if not isinstance(entry, dict) or "value" not in entry:
331 out[key] = entry
332 continue
333 new_entry = dict(entry) # shallow copy preserves metadata
334 # Delegate the per-value rule to redact_value so the snapshot,
335 # the singular GET and the bulk GET can never mask differently.
336 new_entry["value"] = DataSanitizer.redact_value(
337 key,
338 entry.get("ui_element"),
339 entry.get("value"),
340 sensitive_keys,
341 redaction_text,
342 )
343 out[key] = new_entry
344 return out
347# Convenience functions for direct use
348def sanitize_data(data: Any, sensitive_keys: Set[str] | None = None) -> Any:
349 """
350 Remove sensitive keys from data structures.
352 Convenience function that calls DataSanitizer.sanitize().
354 Args:
355 data: The data structure to sanitize
356 sensitive_keys: Optional set of sensitive key names
358 Returns:
359 Sanitized copy of the data
360 """
361 return DataSanitizer.sanitize(data, sensitive_keys)
364def redact_data(
365 data: Any,
366 sensitive_keys: Set[str] | None = None,
367 redaction_text: str = REDACTION_TEXT,
368) -> Any:
369 """
370 Redact (replace) sensitive values in data structures.
372 Convenience function that calls DataSanitizer.redact().
374 Args:
375 data: The data structure to redact
376 sensitive_keys: Optional set of sensitive key names
377 redaction_text: Text to replace sensitive values with
379 Returns:
380 Copy of the data with sensitive values redacted
381 """
382 return DataSanitizer.redact(data, sensitive_keys, redaction_text)
385def filter_research_metadata(research_meta: Any) -> dict:
386 """Filter research_meta to only safe fields for history list API responses.
388 Uses an allowlist approach to prevent leaking settings_snapshot
389 (which contains API keys, passwords, tokens) to the frontend.
390 History list consumers only need is_news_search from metadata.
392 Args:
393 research_meta: Raw research metadata (dict, JSON string, or None)
395 Returns:
396 dict with only safe fields extracted (currently: is_news_search)
397 """
398 try:
399 meta = research_meta or {}
400 if isinstance(meta, str):
401 meta = json.loads(meta)
402 if not isinstance(meta, dict):
403 return {"is_news_search": False}
404 return {
405 "is_news_search": bool(meta.get("is_news_search", False)),
406 }
407 except (json.JSONDecodeError, TypeError, AttributeError):
408 return {"is_news_search": False}
411def strip_settings_snapshot(research_meta: Any) -> dict:
412 """Remove settings_snapshot from research_meta for API responses.
414 settings_snapshot contains all application settings including API keys.
415 This strips it while preserving all other metadata fields that the
416 frontend needs (phase, error_type, processed_query, mode, duration, etc.).
418 Args:
419 research_meta: Raw research metadata (dict, JSON string, or None)
421 Returns:
422 Copy of the dict with settings_snapshot removed
423 """
424 try:
425 meta = research_meta or {}
426 if isinstance(meta, str):
427 meta = json.loads(meta)
428 if not isinstance(meta, dict):
429 return {}
430 return {k: v for k, v in meta.items() if k != "settings_snapshot"}
431 except (json.JSONDecodeError, TypeError, AttributeError):
432 return {}