Coverage for src/local_deep_research/security/log

1"""Sanitize raw strings before writing them to log output.

3``data_sanitizer.py`` handles dict-key redaction (e.g. stripping API keys

4from structured data by key name). This module handles different

5concerns:

7* :func:`strip_control_chars` / :func:`sanitize_for_log` \u2014 make a single

8 string value safe to include in a log line by removing non-printable

9 characters and truncating to a reasonable length.

10* :func:`redact_secrets` \u2014 scrub known sensitive *values* (API keys,

11 passwords, session tokens) from an arbitrary string before it is

12 logged, returned in an error message, or persisted.

13* :func:`sanitize_error_details` \u2014 recurse a structured ``details`` value

14 and redact credential *shapes* from its string leaves before it is

15 serialized to a client. (Value-shape based; contrast

16 ``data_sanitizer.DataSanitizer``, which redacts by key *name*.)

17"""

19import dataclasses

20import re

21from typing import Any, Optional, Union

24# Strip C0/C1 control characters and dangerous Unicode format characters,

25# but preserve visible Unicode (accented, CJK, emoji, etc.)

26_UNSAFE_CHAR_RE = re.compile(

27 r"[\x00-\x1f\x7f-\x9f" # C0/C1 control chars

28 r"\u061c" # Arabic letter mark

29 r"\u200b-\u200f" # Zero-width chars + LTR/RTL marks

30 r"\u202a-\u202e" # Embedding/override (incl. RLO)

31 r"\u2028\u2029" # Line/paragraph separators — forced breaks in rendered

32 # HTML per CSS Text, so a log line carrying one can forge what looks

33 # like a separate entry even though re and str.splitlines() ignore them

34 r"\u2060-\u2064" # Word joiner + math invisible operators

35 r"\u2066-\u2069" # Isolate chars

36 r"\u206a-\u206f" # Digit shape controls

37 r"\ufeff" # BOM / zero-width no-break space

38 r"]"

39)

41# Default minimum length for a value to be considered a redactable secret.

42# Values shorter than this are skipped because a literal ``str.replace`` on

43# a short string would produce false positives in normal message content

44# (e.g. redacting the 3-char string ``key`` would scrub the word "key"

45# everywhere it appears).

46_MIN_SECRET_LENGTH = 8

48# Replacement token written in place of any redacted secret.

49_REDACTION_TOKEN = "***REDACTED***" # noqa: S105 # gitleaks:allow

52def strip_control_chars(value: str) -> str:

53 """Remove control and format characters from *value*, preserving visible Unicode."""

54 return _UNSAFE_CHAR_RE.sub("", value)

57def sanitize_log_record(record) -> None:

58 """loguru patcher stripping control characters from a record's message.

60 loguru holds one patcher per process, so every process that builds its own

61 sink installs this itself. Shared from here rather than from ``log_utils``

62 because that module imports the web stack at module scope, and the MCP

63 subprocess does not.

64 """

65 record["message"] = strip_control_chars(record["message"])

68def sanitize_for_log(value: str, max_length: int = 50) -> str:

69 """Return a log-safe version of *value*.

71 * Control and format characters are stripped; valid Unicode is preserved.

72 * The result is truncated to *max_length* characters.

73 """

74 cleaned = strip_control_chars(value)

75 if len(cleaned) > max_length:

76 cleaned = (

77 cleaned[: max_length - 3] + "..."

78 if max_length > 3

79 else cleaned[:max_length]

80 )

81 return cleaned

84def redact_secrets(

85 message: str,

86 *secrets: Optional[str],

87 min_length: int = _MIN_SECRET_LENGTH,

88 replacement: str = _REDACTION_TOKEN,

89) -> str:

90 """Replace each occurrence of any *secret* in *message* with *replacement*.

92 Use this before writing a string to a log sink, returning it in an

93 error response, or persisting it \u2014 when the string may have been

94 constructed from upstream exception messages, URLs, or other

95 sources that could contain a value the caller already knows is

96 sensitive.

98 Each *secret* is matched as a literal substring (``str.replace``).

99 The function does not normalize encodings: if a secret appears

100 URL-encoded or otherwise transformed in *message*, the transformed

101 form is NOT redacted unless the caller also passes that

102 transformed form.

103

104 When multiple secrets are passed, they are applied in descending

105 length order so a shorter secret that happens to be a substring of

106 a longer one cannot consume part of the longer match. Example:

107 given secrets ``"abc12345"`` and ``"sk-abc12345"``, the longer one

108 is replaced first.

109

110 Args:

111 message: The string to scrub. Returned unchanged if falsy.

112 *secrets: Zero or more candidate secret values. ``None`` and

113 values shorter than *min_length* are silently skipped \u2014 the

114 caller is responsible for noticing missing config.

115 min_length: Minimum secret length to redact. Values shorter than

116 this are skipped to avoid corrupting normal message content

117 (a 1- or 2-character secret would match too aggressively).

118 Defaults to 8. Real API keys and session tokens are

119 typically 16+ characters.

120 replacement: String written in place of each redacted secret.

121 Defaults to ``"***REDACTED***"``.

122

123 Returns:

124 *message* with every occurrence of each qualifying secret

125 replaced.

126

127 See ``tests/security/test_log_sanitizer.py::TestRedactSecrets`` for

128 worked examples (doctest examples are omitted because the

129 repository's gitleaks rule flags any token-shaped literal in

130 docstrings).

131 """

132 if not message:

133 return message

134 # Longest-first prevents a shorter overlapping secret from

135 # truncating a longer one once the replacement token is in place.

136 ordered = sorted(

137 (s for s in secrets if s and len(s) >= min_length),

138 key=len,

139 reverse=True,

140 )

141 for secret in ordered:

142 message = message.replace(secret, replacement)

143 return message

144

145

146# Pre-compiled regex patterns for common credential formats found in HTTP

147# library exception messages. Used by sanitize_error_message().

148#

149# Order matters: the URL-credentials pattern must run BEFORE the URL-param

150# pattern. Otherwise an input like ``?api-key=https://user:pass@host`` gets

151# its ``https`` consumed by the param replacement, the credentials pattern

152# no longer matches, and ``user:pass`` leaks.

153_CREDENTIAL_PATTERNS: list[tuple[re.Pattern[str], str]] = [

154 # Bearer tokens

155 (re.compile(r"Bearer\s+[A-Za-z0-9\-._~+/]+=*"), "Bearer [REDACTED]"),

156 # Authorization header WITH an explicit scheme. The scheme word is a

157 # strong anchor that rules out prose, so redact the credential on length

158 # alone (>=8 chars) regardless of its shape — this catches even an

159 # all-alphabetic Basic/Digest value. The scheme is preserved for

160 # debuggability. (A short prose word after a scheme is rare and would

161 # only be over-redacted, never leaked.)

162 (

163 re.compile(

164 r"(?i)(authorization\s*[:=]\s*)"

166 r"[A-Za-z0-9\-._~+/]{8,}=*"

167 ),

168 r"\1\2 [REDACTED]",

169 ),

170 # Authorization header WITHOUT a scheme — here the value could be prose

171 # ("Authorization: required"), so require a *token-shaped* value (>=8

172 # chars containing a digit/+///=/_) to catch a raw token while leaving

173 # all-alphabetic prose intact.

174 (

175 re.compile(

176 r"(?i)(authorization\s*[:=]\s*)"

177 r"(?=[A-Za-z0-9\-._~+/]*[0-9+/=_])"

178 r"[A-Za-z0-9\-._~+/]{8,}=*"

179 ),

180 r"\1[REDACTED]",

181 ),

182 # x-api-key header — the label is a strong anchor (it doesn't appear in

183 # ordinary prose), so redact any sufficiently long value (>=16 chars)

184 # regardless of shape. Short values like "invalid"/"missing" stay intact.

185 (

186 re.compile(r"(?i)(x-api-key\s*[:=]\s*)[A-Za-z0-9\-._~+/]{16,}=*"),

187 r"\1[REDACTED]",

188 ),

189 # URL credentials (user:pass@host) in ANY URL scheme. Userinfo in a URL is

190 # always a credential, so this is not restricted to http(s): it also covers

191 # URL-form database connection strings — including SQLAlchemy's

192 # ``dialect+driver`` form (``postgresql+psycopg2://``, ``mysql+pymysql://``,

193 # ``mongodb+srv://``) and password-only DSNs (``redis://:pass@host``) —

194 # which are the most common credential-bearing strings in a raw DB/driver

195 # exception message. (Key=value DSNs like pyodbc's ``Server=...;Pwd=...``

196 # have no ``://`` and are out of scope for a userinfo regex.)

197 #

198 # The scheme is matched case-insensitively (URL schemes are case-insensitive

199 # per RFC 3986). NOTE: do NOT re-add a leading ``\b`` here — the scheme's

200 # first char is a word char, so ``\b`` fails to anchor when the URL is glued

201 # to a preceding word char (``Xhttps://user:pass@``) and the credential then

202 # leaks. The ``://`` literal plus the leading-letter requirement are already

203 # strong anchors against prose. The trailing ``@`` is required and ``/`` is

204 # excluded from the userinfo, so a credential-less DSN with a port

205 # (``postgresql://host:5432/db``) does not match.

206 (

207 re.compile(

208 r"([A-Za-z][A-Za-z0-9+.\-]{1,31}://)([^:\s/@]*):([^@\s/]+)@"

209 ),

210 r"\1[REDACTED]:[REDACTED]@",

211 ),

212 # Credential-bearing URL query parameters (?api_key=..., &access_token=...).

213 # Specific multi-word names precede the short catch-alls so the full

214 # parameter name is matched (e.g. ``secret_key`` not just ``secret``).

215 (

216 re.compile(

217 r"(?i)([?&])("

218 r"api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|"

219 r"auth[_-]?token|session[_-]?token|secret[_-]?key|bearer[_-]?token|"

220 r"subscription[_-]?key|client[_-]?secret|api[_-]?secret|"

221 r"app[_-]?secret|private[_-]?key|"

223 r")=([^&\s#]+)"

224 ),

225 r"\1\2=[REDACTED]",

226 ),

227 # Common API key prefixes (sk-*, pk-*) — includes hyphens for modern

228 # formats like sk-proj-... and sk-ant-api03-...

229 (re.compile(r"\b(sk-[A-Za-z0-9\-]{20,})\b"), "[REDACTED_KEY]"),

230 (re.compile(r"\b(pk-[A-Za-z0-9\-]{20,})\b"), "[REDACTED_KEY]"),

231 # Google API keys (AIza...) — match generously to cover length variants

232 # while the 20-char floor avoids short false positives.

233 (re.compile(r"\bAIza[0-9A-Za-z\-_]{20,}\b"), "[REDACTED_KEY]"),

234 # Distinctive provider token prefixes. These mirror the canonical,

235 # actively-maintained gitleaks ruleset (https://github.com/gitleaks/

236 # gitleaks, config/gitleaks.toml) — refresh from there when new token

237 # formats appear. They are prefix-anchored (very low false-positive risk

238 # in prose); the dual-scrub redact_secrets(known_literal) path remains the

239 # backstop for arbitrary/unknown secret shapes. See

240 # docs/developing/credential-scrubbing.md for the maintenance process.

241 # GitHub tokens: ghp_/gho_/ghu_/ghs_/ghr_ + fine-grained PATs.

242 (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"), "[REDACTED_KEY]"),

243 (re.compile(r"\bgithub_pat_[A-Za-z0-9_]{30,}\b"), "[REDACTED_KEY]"),

244 # AWS access key IDs (AKIA/ASIA/ABIA/ACCA/A3T...).

245 # Accepted false positive (over-redaction is the safe failure): a

246 # contiguous 20-char all-caps word starting with one of these prefixes

247 # is redacted. Mirrors gitleaks exactly.

248 (

249 re.compile(r"\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b"),

250 "[REDACTED_KEY]",

251 ),

252 # Slack tokens: xox[baeprs]-* (bot/user/app/refresh/config-refresh/...)

253 # and app-level xapp-* tokens. The ``xapp``/``xox`` prefixes are generic

254 # enough to appear in hyphenated identifiers, so (unlike the distinctive

255 # GitHub/AWS prefixes) we additionally require the long numeric workspace

256 # ID that every real Slack token carries (gitleaks expects ``[0-9]{10,13}``

257 # segments). This keeps real tokens redacted while leaving prose such as

258 # ``xapp-release-notes-2026`` intact.

259 (

260 re.compile(

261 r"\bxox[baeprs]-(?=[A-Za-z0-9-]*[0-9]{9,})[A-Za-z0-9-]{10,}\b"

262 ),

263 "[REDACTED_KEY]",

264 ),

265 (

266 re.compile(r"\bxapp-(?=[A-Za-z0-9-]*[0-9]{9,})[A-Za-z0-9-]{10,}\b"),

267 "[REDACTED_KEY]",

268 ),

269 # Google OAuth access tokens (ya29...).

270 (re.compile(r"\bya29\.[A-Za-z0-9_\-]{20,}"), "[REDACTED_KEY]"),

271 # JSON Web Tokens (three base64url segments). The two literal dots make

272 # this distinctive enough to avoid prose false positives. Accepted FP:

273 # a 3-part dotted identifier whose segments start with "eyJ" and are

274 # >=8 base64url chars (e.g. "eyJsonParser.eyJsonReader.eyJsonX") is

275 # redacted — over-redaction, not a leak. ``/`` is intentionally omitted

276 # (RFC 7515 JWTs are base64url); ``Bearer``/``Authorization`` paths

277 # already catch standard-base64 JWTs.

278 (

279 re.compile(

280 r"\beyJ[A-Za-z0-9_+\-]{8,}\.[A-Za-z0-9_+\-]{8,}\.[A-Za-z0-9_+\-]+"

281 ),

282 "[REDACTED_KEY]",

283 ),

284]

285

286

287def sanitize_error_message(message: str) -> str:

288 """Remove or mask API keys, tokens, and secrets from *message* using

289 pattern matching for common credential formats.

290

291 Use this as a first scrub pass on exception messages before logging,

292 followed by :func:`redact_secrets` with known literal values (the

293 "dual-scrub" pattern).

294

295 Handles:

296 * Bearer tokens (``Bearer sk-...``)

297 * ``Authorization:`` (any scheme) and ``x-api-key:`` headers

298 * URL query parameters (``?api_key=``, ``?access_token=``,

299 ``?refresh_token=``, ``?subscription-key=``, ``?secret=``, ...)

300 * URL-embedded credentials (``https://user:pass@host``)

301 * Well-known token prefixes — ``sk-``/``pk-``, Google ``AIza``/``ya29.``,

302 GitHub ``ghp_``/``github_pat_``, AWS ``AKIA``/``ASIA``, Slack ``xox*-``,

303 and JWTs (``eyJ….….…``). See ``docs/developing/credential-scrubbing.md``.

304 """

305 if not message:

306 return message

307 for pattern, replacement in _CREDENTIAL_PATTERNS:

308 message = pattern.sub(replacement, message)

309 return message

310

311

312def scrub_error(error: Union[BaseException, str], *secrets: Any) -> str:

313 """Return a log/DB-safe rendering of *error* (the "dual-scrub").

314

315 Composes the two scrub passes every catch site needs:

316 :func:`sanitize_error_message` (catches credential *shapes* — Bearer

317 tokens, URL-embedded credentials, ``sk-``/``pk-`` keys) followed by

318 :func:`redact_secrets` with the caller's known literal secret values.

319

320 Use this at every catch site that logs or persists an exception so

321 the two passes can never drift apart per-site.

322 ``BaseSearchEngine._scrub_error`` delegates here, resolving its

323 engine's ``_secret_attrs`` into the *secrets* arguments.

324

325 Defensive by design: this runs inside ``except`` blocks, so it must

326 never raise. ``str(error)`` is guarded (a custom exception whose

327 ``__str__`` raises won't crash the handler) and each secret is coerced

328 to ``str`` (a misconfigured non-string secret, e.g. an int from

329 settings, won't trip ``redact_secrets``' ``len()`` check).

330

331 Args:

332 error: An exception or a pre-built message string.

333 *secrets: Known literal secret values to redact. ``None`` and

334 falsy values are silently skipped.

335

336 Returns:

337 The scrubbed message, safe for production log sinks.

338 """

339 try:

340 message = str(error)

341 except Exception:

342 message = f"<unprintable {type(error).__name__}>"

343 # Coerce truthy non-str secrets to str; keep None/falsy as-is

344 # (redact_secrets filters those out). Guarded per secret: a

345 # pathological value whose __bool__/__str__ raises cannot be

346 # literal-matched anyway, so it is skipped rather than allowed to

347 # crash the except handler this runs in.

348 safe_secrets = []

349 for v in secrets:

350 try:

351 safe_secrets.append(v and str(v))

352 except Exception:

353 continue

354 return redact_secrets(sanitize_error_message(message), *safe_secrets)

355

356

357def sanitize_error_for_client(message: str, max_length: int = 200) -> str:

358 """Make an exception-derived string safe to return to an HTTP client.

359

360 Composes :func:`sanitize_error_message` (credential redaction) and

361 :func:`sanitize_for_log` (control-char strip + length cap). Credential

362 scrubbing runs FIRST, on the full untruncated string, so a secret near

363 the ``max_length`` boundary cannot be split by truncation and slip past

364 the regexes.

365

366 Use this for any exception text surfaced to the browser (API/JSON/SSE

367 responses); keep the raw exception server-side via ``logger.exception``.

368 """

369 return sanitize_for_log(

370 sanitize_error_message(message), max_length=max_length

371 )

372

373

374def sanitize_error_details(value: Any) -> Any:

375 """Recursively redact credential *shapes* from the string leaves of a

376 structured ``details`` value (``dict`` / ``list`` / ``tuple`` / dataclass).

377

378 Intended for the ``details`` payload of an exception ``to_dict()`` that is

379 serialized to a client (e.g. ``NewsAPIException`` / ``WebAPIException``).

380 This is the *value-shape* counterpart to :class:`data_sanitizer.DataSanitizer`,

381 which redacts by *key name*; use this one when the concern is a credential

382 embedded anywhere in the text, regardless of the key it sits under.

383

384 Behaviour by node type:

385

386 * ``dict`` — recurse into values; ``str`` keys are also run through

387 :func:`sanitize_error_message` (a credential used *as* a key would

388 otherwise ship verbatim as a JSON key). Two distinct keys that both

389 redact to the same token collapse to one entry — an acceptable

390 fidelity loss for the pathological case of credential-shaped keys.

391 * ``list`` / ``tuple`` — both rebuilt as a plain ``list``: the payload is

392 about to be JSON-serialized (a tuple already becomes a JSON array), and

393 rebuilding via ``type(value)(<generator>)`` would raise on a namedtuple /

394 tuple subclass whose constructor is not ``(iterable) -> instance`` —

395 inside a Flask error handler that degrades a structured error into a bare

396 500.

397 * **dataclass instance** — converted via ``dataclasses.asdict`` and recursed.

398 Flask's default JSON provider serializes a dataclass (via ``asdict``), so

399 without this a credential in a dataclass field would ship un-redacted.

400 * ``str`` leaf — redacted via :func:`sanitize_error_message` (credential-shape

401 redaction only — no length cap or control-char strip, so structured values

402 such as IDs survive).

403 * anything else (ints, bools, ``None``, and any other object) — passed

404 through untouched. Note some passthrough types (``set``/``frozenset``,

405 ``bytes``, a non-``str``-mixin ``Enum``) are not JSON-serializable, so a

406 credential inside them fails *closed* at ``jsonify`` (generic 500) rather

407 than leaking.

408

409 Redaction is shape-based, so it never removes a value by key name — a benign

410 ``{"query": "reset my password"}`` is unchanged. Containers are rebuilt fresh

411 (the input is not mutated); passthrough leaf objects are returned by

412 reference.

413 """

414 if isinstance(value, dict):

415 return {

416 (sanitize_error_message(k) if isinstance(k, str) else k): (

417 sanitize_error_details(v)

418 )

419 for k, v in value.items()

420 }

421 if isinstance(value, (list, tuple)):

422 return [sanitize_error_details(v) for v in value]

423 # dataclass *instance* (not the class itself) — Flask serializes it via

424 # asdict(), so recurse into its fields to redact any credential-shaped one.

425 if dataclasses.is_dataclass(value) and not isinstance(value, type):

426 return sanitize_error_details(dataclasses.asdict(value))

427 if isinstance(value, str):

428 return sanitize_error_message(value)

429 return value

Coverage for src/local_deep_research/security/log_sanitizer.py: 100%

53 statements