Coverage for src/local_deep_research/security/log

1"""Sanitize raw strings before writing them to log output.

3``data_sanitizer.py`` handles dict-key redaction (e.g. stripping API keys

4from structured data by key name). This module handles different

5concerns:

7* :func:`strip_control_chars` / :func:`sanitize_for_log` \u2014 make a single

8 string value safe to include in a log line by removing non-printable

9 characters and truncating to a reasonable length.

10* :func:`redact_secrets` \u2014 scrub known sensitive *values* (API keys,

11 passwords, session tokens) from an arbitrary string before it is

12 logged, returned in an error message, or persisted.

13* :func:`sanitize_error_details` \u2014 recurse a structured ``details`` value

14 and redact credential *shapes* from its string leaves before it is

15 serialized to a client. (Value-shape based; contrast

16 ``data_sanitizer.DataSanitizer``, which redacts by key *name*.)

17"""

19import dataclasses

20import re

21from typing import Any, Optional, Union

24# Strip C0/C1 control characters and dangerous Unicode format characters,

25# but preserve visible Unicode (accented, CJK, emoji, etc.)

26_UNSAFE_CHAR_RE = re.compile(

27 r"[\x00-\x1f\x7f-\x9f" # C0/C1 control chars

28 r"\u061c" # Arabic letter mark

29 r"\u200b-\u200f" # Zero-width chars + LTR/RTL marks

30 r"\u202a-\u202e" # Embedding/override (incl. RLO)

31 r"\u2060-\u2064" # Word joiner + math invisible operators

32 r"\u2066-\u2069" # Isolate chars

33 r"\u206a-\u206f" # Digit shape controls

34 r"\ufeff" # BOM / zero-width no-break space

35 r"]"

36)

38# Default minimum length for a value to be considered a redactable secret.

39# Values shorter than this are skipped because a literal ``str.replace`` on

40# a short string would produce false positives in normal message content

41# (e.g. redacting the 3-char string ``key`` would scrub the word "key"

42# everywhere it appears).

43_MIN_SECRET_LENGTH = 8

45# Replacement token written in place of any redacted secret.

46_REDACTION_TOKEN = "***REDACTED***" # noqa: S105 # gitleaks:allow

49def strip_control_chars(value: str) -> str:

50 """Remove control and format characters from *value*, preserving visible Unicode."""

51 return _UNSAFE_CHAR_RE.sub("", value)

54def sanitize_for_log(value: str, max_length: int = 50) -> str:

55 """Return a log-safe version of *value*.

57 * Control and format characters are stripped; valid Unicode is preserved.

58 * The result is truncated to *max_length* characters.

59 """

60 cleaned = strip_control_chars(value)

61 if len(cleaned) > max_length:

62 cleaned = (

63 cleaned[: max_length - 3] + "..."

64 if max_length > 3

65 else cleaned[:max_length]

66 )

67 return cleaned

70def redact_secrets(

71 message: str,

72 *secrets: Optional[str],

73 min_length: int = _MIN_SECRET_LENGTH,

74 replacement: str = _REDACTION_TOKEN,

75) -> str:

76 """Replace each occurrence of any *secret* in *message* with *replacement*.

78 Use this before writing a string to a log sink, returning it in an

79 error response, or persisting it \u2014 when the string may have been

80 constructed from upstream exception messages, URLs, or other

81 sources that could contain a value the caller already knows is

82 sensitive.

84 Each *secret* is matched as a literal substring (``str.replace``).

85 The function does not normalize encodings: if a secret appears

86 URL-encoded or otherwise transformed in *message*, the transformed

87 form is NOT redacted unless the caller also passes that

88 transformed form.

90 When multiple secrets are passed, they are applied in descending

91 length order so a shorter secret that happens to be a substring of

92 a longer one cannot consume part of the longer match. Example:

93 given secrets ``"abc12345"`` and ``"sk-abc12345"``, the longer one

94 is replaced first.

96 Args:

97 message: The string to scrub. Returned unchanged if falsy.

98 *secrets: Zero or more candidate secret values. ``None`` and

99 values shorter than *min_length* are silently skipped \u2014 the

100 caller is responsible for noticing missing config.

101 min_length: Minimum secret length to redact. Values shorter than

102 this are skipped to avoid corrupting normal message content

103 (a 1- or 2-character secret would match too aggressively).

104 Defaults to 8. Real API keys and session tokens are

105 typically 16+ characters.

106 replacement: String written in place of each redacted secret.

107 Defaults to ``"***REDACTED***"``.

108

109 Returns:

110 *message* with every occurrence of each qualifying secret

111 replaced.

112

113 See ``tests/security/test_log_sanitizer.py::TestRedactSecrets`` for

114 worked examples (doctest examples are omitted because the

115 repository's gitleaks rule flags any token-shaped literal in

116 docstrings).

117 """

118 if not message:

119 return message

120 # Longest-first prevents a shorter overlapping secret from

121 # truncating a longer one once the replacement token is in place.

122 ordered = sorted(

123 (s for s in secrets if s and len(s) >= min_length),

124 key=len,

125 reverse=True,

126 )

127 for secret in ordered:

128 message = message.replace(secret, replacement)

129 return message

130

131

132# Pre-compiled regex patterns for common credential formats found in HTTP

133# library exception messages. Used by sanitize_error_message().

134#

135# Order matters: the URL-credentials pattern must run BEFORE the URL-param

136# pattern. Otherwise an input like ``?api-key=https://user:pass@host`` gets

137# its ``https`` consumed by the param replacement, the credentials pattern

138# no longer matches, and ``user:pass`` leaks.

139_CREDENTIAL_PATTERNS: list[tuple[re.Pattern[str], str]] = [

140 # Bearer tokens

141 (re.compile(r"Bearer\s+[A-Za-z0-9\-._~+/]+=*"), "Bearer [REDACTED]"),

142 # Authorization header WITH an explicit scheme. The scheme word is a

143 # strong anchor that rules out prose, so redact the credential on length

144 # alone (>=8 chars) regardless of its shape — this catches even an

145 # all-alphabetic Basic/Digest value. The scheme is preserved for

146 # debuggability. (A short prose word after a scheme is rare and would

147 # only be over-redacted, never leaked.)

148 (

149 re.compile(

150 r"(?i)(authorization\s*[:=]\s*)"

152 r"[A-Za-z0-9\-._~+/]{8,}=*"

153 ),

154 r"\1\2 [REDACTED]",

155 ),

156 # Authorization header WITHOUT a scheme — here the value could be prose

157 # ("Authorization: required"), so require a *token-shaped* value (>=8

158 # chars containing a digit/+///=/_) to catch a raw token while leaving

159 # all-alphabetic prose intact.

160 (

161 re.compile(

162 r"(?i)(authorization\s*[:=]\s*)"

163 r"(?=[A-Za-z0-9\-._~+/]*[0-9+/=_])"

164 r"[A-Za-z0-9\-._~+/]{8,}=*"

165 ),

166 r"\1[REDACTED]",

167 ),

168 # x-api-key header — the label is a strong anchor (it doesn't appear in

169 # ordinary prose), so redact any sufficiently long value (>=16 chars)

170 # regardless of shape. Short values like "invalid"/"missing" stay intact.

171 (

172 re.compile(r"(?i)(x-api-key\s*[:=]\s*)[A-Za-z0-9\-._~+/]{16,}=*"),

173 r"\1[REDACTED]",

174 ),

175 # URL credentials (user:pass@host) in ANY URL scheme. Userinfo in a URL is

176 # always a credential, so this is not restricted to http(s): it also covers

177 # URL-form database connection strings — including SQLAlchemy's

178 # ``dialect+driver`` form (``postgresql+psycopg2://``, ``mysql+pymysql://``,

179 # ``mongodb+srv://``) and password-only DSNs (``redis://:pass@host``) —

180 # which are the most common credential-bearing strings in a raw DB/driver

181 # exception message. (Key=value DSNs like pyodbc's ``Server=...;Pwd=...``

182 # have no ``://`` and are out of scope for a userinfo regex.)

183 #

184 # The scheme is matched case-insensitively (URL schemes are case-insensitive

185 # per RFC 3986). NOTE: do NOT re-add a leading ``\b`` here — the scheme's

186 # first char is a word char, so ``\b`` fails to anchor when the URL is glued

187 # to a preceding word char (``Xhttps://user:pass@``) and the credential then

188 # leaks. The ``://`` literal plus the leading-letter requirement are already

189 # strong anchors against prose. The trailing ``@`` is required and ``/`` is

190 # excluded from the userinfo, so a credential-less DSN with a port

191 # (``postgresql://host:5432/db``) does not match.

192 (

193 re.compile(

194 r"([A-Za-z][A-Za-z0-9+.\-]{1,31}://)([^:\s/@]*):([^@\s/]+)@"

195 ),

196 r"\1[REDACTED]:[REDACTED]@",

197 ),

198 # Credential-bearing URL query parameters (?api_key=..., &access_token=...).

199 # Specific multi-word names precede the short catch-alls so the full

200 # parameter name is matched (e.g. ``secret_key`` not just ``secret``).

201 (

202 re.compile(

203 r"(?i)([?&])("

204 r"api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|"

205 r"auth[_-]?token|session[_-]?token|secret[_-]?key|bearer[_-]?token|"

206 r"subscription[_-]?key|client[_-]?secret|api[_-]?secret|"

207 r"app[_-]?secret|private[_-]?key|"

209 r")=([^&\s#]+)"

210 ),

211 r"\1\2=[REDACTED]",

212 ),

213 # Common API key prefixes (sk-*, pk-*) — includes hyphens for modern

214 # formats like sk-proj-... and sk-ant-api03-...

215 (re.compile(r"\b(sk-[A-Za-z0-9\-]{20,})\b"), "[REDACTED_KEY]"),

216 (re.compile(r"\b(pk-[A-Za-z0-9\-]{20,})\b"), "[REDACTED_KEY]"),

217 # Google API keys (AIza...) — match generously to cover length variants

218 # while the 20-char floor avoids short false positives.

219 (re.compile(r"\bAIza[0-9A-Za-z\-_]{20,}\b"), "[REDACTED_KEY]"),

220 # Distinctive provider token prefixes. These mirror the canonical,

221 # actively-maintained gitleaks ruleset (https://github.com/gitleaks/

222 # gitleaks, config/gitleaks.toml) — refresh from there when new token

223 # formats appear. They are prefix-anchored (very low false-positive risk

224 # in prose); the dual-scrub redact_secrets(known_literal) path remains the

225 # backstop for arbitrary/unknown secret shapes. See

226 # docs/developing/credential-scrubbing.md for the maintenance process.

227 # GitHub tokens: ghp_/gho_/ghu_/ghs_/ghr_ + fine-grained PATs.

228 (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"), "[REDACTED_KEY]"),

229 (re.compile(r"\bgithub_pat_[A-Za-z0-9_]{30,}\b"), "[REDACTED_KEY]"),

230 # AWS access key IDs (AKIA/ASIA/ABIA/ACCA/A3T...).

231 # Accepted false positive (over-redaction is the safe failure): a

232 # contiguous 20-char all-caps word starting with one of these prefixes

233 # is redacted. Mirrors gitleaks exactly.

234 (

235 re.compile(r"\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b"),

236 "[REDACTED_KEY]",

237 ),

238 # Slack tokens: xox[baeprs]-* (bot/user/app/refresh/config-refresh/...)

239 # and app-level xapp-* tokens. The ``xapp``/``xox`` prefixes are generic

240 # enough to appear in hyphenated identifiers, so (unlike the distinctive

241 # GitHub/AWS prefixes) we additionally require the long numeric workspace

242 # ID that every real Slack token carries (gitleaks expects ``[0-9]{10,13}``

243 # segments). This keeps real tokens redacted while leaving prose such as

244 # ``xapp-release-notes-2026`` intact.

245 (

246 re.compile(

247 r"\bxox[baeprs]-(?=[A-Za-z0-9-]*[0-9]{9,})[A-Za-z0-9-]{10,}\b"

248 ),

249 "[REDACTED_KEY]",

250 ),

251 (

252 re.compile(r"\bxapp-(?=[A-Za-z0-9-]*[0-9]{9,})[A-Za-z0-9-]{10,}\b"),

253 "[REDACTED_KEY]",

254 ),

255 # Google OAuth access tokens (ya29...).

256 (re.compile(r"\bya29\.[A-Za-z0-9_\-]{20,}"), "[REDACTED_KEY]"),

257 # JSON Web Tokens (three base64url segments). The two literal dots make

258 # this distinctive enough to avoid prose false positives. Accepted FP:

259 # a 3-part dotted identifier whose segments start with "eyJ" and are

260 # >=8 base64url chars (e.g. "eyJsonParser.eyJsonReader.eyJsonX") is

261 # redacted — over-redaction, not a leak. ``/`` is intentionally omitted

262 # (RFC 7515 JWTs are base64url); ``Bearer``/``Authorization`` paths

263 # already catch standard-base64 JWTs.

264 (

265 re.compile(

266 r"\beyJ[A-Za-z0-9_+\-]{8,}\.[A-Za-z0-9_+\-]{8,}\.[A-Za-z0-9_+\-]+"

267 ),

268 "[REDACTED_KEY]",

269 ),

270]

271

272

273def sanitize_error_message(message: str) -> str:

274 """Remove or mask API keys, tokens, and secrets from *message* using

275 pattern matching for common credential formats.

276

277 Use this as a first scrub pass on exception messages before logging,

278 followed by :func:`redact_secrets` with known literal values (the

279 "dual-scrub" pattern).

280

281 Handles:

282 * Bearer tokens (``Bearer sk-...``)

283 * ``Authorization:`` (any scheme) and ``x-api-key:`` headers

284 * URL query parameters (``?api_key=``, ``?access_token=``,

285 ``?refresh_token=``, ``?subscription-key=``, ``?secret=``, ...)

286 * URL-embedded credentials (``https://user:pass@host``)

287 * Well-known token prefixes — ``sk-``/``pk-``, Google ``AIza``/``ya29.``,

288 GitHub ``ghp_``/``github_pat_``, AWS ``AKIA``/``ASIA``, Slack ``xox*-``,

289 and JWTs (``eyJ….….…``). See ``docs/developing/credential-scrubbing.md``.

290 """

291 if not message:

292 return message

293 for pattern, replacement in _CREDENTIAL_PATTERNS:

294 message = pattern.sub(replacement, message)

295 return message

296

297

298def scrub_error(error: Union[BaseException, str], *secrets: Any) -> str:

299 """Return a log/DB-safe rendering of *error* (the "dual-scrub").

300

301 Composes the two scrub passes every catch site needs:

302 :func:`sanitize_error_message` (catches credential *shapes* — Bearer

303 tokens, URL-embedded credentials, ``sk-``/``pk-`` keys) followed by

304 :func:`redact_secrets` with the caller's known literal secret values.

305

306 Use this at every catch site that logs or persists an exception so

307 the two passes can never drift apart per-site.

308 ``BaseSearchEngine._scrub_error`` delegates here, resolving its

309 engine's ``_secret_attrs`` into the *secrets* arguments.

310

311 Defensive by design: this runs inside ``except`` blocks, so it must

312 never raise. ``str(error)`` is guarded (a custom exception whose

313 ``__str__`` raises won't crash the handler) and each secret is coerced

314 to ``str`` (a misconfigured non-string secret, e.g. an int from

315 settings, won't trip ``redact_secrets``' ``len()`` check).

316

317 Args:

318 error: An exception or a pre-built message string.

319 *secrets: Known literal secret values to redact. ``None`` and

320 falsy values are silently skipped.

321

322 Returns:

323 The scrubbed message, safe for production log sinks.

324 """

325 try:

326 message = str(error)

327 except Exception:

328 message = f"<unprintable {type(error).__name__}>"

329 # Coerce truthy non-str secrets to str; keep None/falsy as-is

330 # (redact_secrets filters those out). Guarded per secret: a

331 # pathological value whose __bool__/__str__ raises cannot be

332 # literal-matched anyway, so it is skipped rather than allowed to

333 # crash the except handler this runs in.

334 safe_secrets = []

335 for v in secrets:

336 try:

337 safe_secrets.append(v and str(v))

338 except Exception:

339 continue

340 return redact_secrets(sanitize_error_message(message), *safe_secrets)

341

342

343def sanitize_error_for_client(message: str, max_length: int = 200) -> str:

344 """Make an exception-derived string safe to return to an HTTP client.

345

346 Composes :func:`sanitize_error_message` (credential redaction) and

347 :func:`sanitize_for_log` (control-char strip + length cap). Credential

348 scrubbing runs FIRST, on the full untruncated string, so a secret near

349 the ``max_length`` boundary cannot be split by truncation and slip past

350 the regexes.

351

352 Use this for any exception text surfaced to the browser (API/JSON/SSE

353 responses); keep the raw exception server-side via ``logger.exception``.

354 """

355 return sanitize_for_log(

356 sanitize_error_message(message), max_length=max_length

357 )

358

359

360def sanitize_error_details(value: Any) -> Any:

361 """Recursively redact credential *shapes* from the string leaves of a

362 structured ``details`` value (``dict`` / ``list`` / ``tuple`` / dataclass).

363

364 Intended for the ``details`` payload of an exception ``to_dict()`` that is

365 serialized to a client (e.g. ``NewsAPIException`` / ``WebAPIException``).

366 This is the *value-shape* counterpart to :class:`data_sanitizer.DataSanitizer`,

367 which redacts by *key name*; use this one when the concern is a credential

368 embedded anywhere in the text, regardless of the key it sits under.

369

370 Behaviour by node type:

371

372 * ``dict`` — recurse into values; ``str`` keys are also run through

373 :func:`sanitize_error_message` (a credential used *as* a key would

374 otherwise ship verbatim as a JSON key). Two distinct keys that both

375 redact to the same token collapse to one entry — an acceptable

376 fidelity loss for the pathological case of credential-shaped keys.

377 * ``list`` / ``tuple`` — both rebuilt as a plain ``list``: the payload is

378 about to be JSON-serialized (a tuple already becomes a JSON array), and

379 rebuilding via ``type(value)(<generator>)`` would raise on a namedtuple /

380 tuple subclass whose constructor is not ``(iterable) -> instance`` —

381 inside a Flask error handler that degrades a structured error into a bare

382 500.

383 * **dataclass instance** — converted via ``dataclasses.asdict`` and recursed.

384 Flask's default JSON provider serializes a dataclass (via ``asdict``), so

385 without this a credential in a dataclass field would ship un-redacted.

386 * ``str`` leaf — redacted via :func:`sanitize_error_message` (credential-shape

387 redaction only — no length cap or control-char strip, so structured values

388 such as IDs survive).

389 * anything else (ints, bools, ``None``, and any other object) — passed

390 through untouched. Note some passthrough types (``set``/``frozenset``,

391 ``bytes``, a non-``str``-mixin ``Enum``) are not JSON-serializable, so a

392 credential inside them fails *closed* at ``jsonify`` (generic 500) rather

393 than leaking.

394

395 Redaction is shape-based, so it never removes a value by key name — a benign

396 ``{"query": "reset my password"}`` is unchanged. Containers are rebuilt fresh

397 (the input is not mutated); passthrough leaf objects are returned by

398 reference.

399 """

400 if isinstance(value, dict):

401 return {

402 (sanitize_error_message(k) if isinstance(k, str) else k): (

403 sanitize_error_details(v)

404 )

405 for k, v in value.items()

406 }

407 if isinstance(value, (list, tuple)):

408 return [sanitize_error_details(v) for v in value]

409 # dataclass *instance* (not the class itself) — Flask serializes it via

410 # asdict(), so recurse into its fields to redact any credential-shaped one.

411 if dataclasses.is_dataclass(value) and not isinstance(value, type):

412 return sanitize_error_details(dataclasses.asdict(value))

413 if isinstance(value, str):

414 return sanitize_error_message(value)

415 return value

Coverage for src/local_deep_research/security/log_sanitizer.py: 100%

51 statements