Coverage for src/local_deep_research/security/log_sanitizer.py: 93%
1592 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Sanitize raw strings before writing them to log output.
3``data_sanitizer.py`` handles dict-key redaction (e.g. stripping API keys
4from structured data by key name). This module handles different
5concerns:
7* :func:`strip_control_chars` / :func:`sanitize_for_log` \u2014 make a single
8 string value safe to include in a log line by removing non-printable
9 characters and truncating to a reasonable length.
10 :func:`redact_and_bound_for_log` redacts credential shapes before it
11 truncates, for a value whose cut could otherwise hide a credential from
12 the log patcher's redaction.
13* :func:`redact_secrets` \u2014 scrub known sensitive *values* (API keys,
14 passwords, session tokens) from an arbitrary string before it is
15 logged, returned in an error message, or persisted.
16* :func:`sanitize_error_details` \u2014 recurse a structured ``details`` value
17 and redact credential *shapes* from its string leaves before it is
18 serialized to a client. (Value-shape based; contrast
19 ``data_sanitizer.DataSanitizer``, which redacts by key *name*.)
21Threat model of the log-record redaction (:func:`redact_log_record`):
22exception classes and the code that logs are application code, and are
23trusted; the *data* inside exceptions -- messages, ``args``, fields and
24notes built from upstream responses, URLs or user input -- and bound
25``extra`` values are not. Credential-shaped data in exception text,
26``args``, notes and ``extra`` (known key and token shapes, URL userinfo,
27sensitive ``name=value`` pairs, and ``extra`` values under sensitive key
28names) is redacted before any sink renders it, and every record is
29picklable by an ``enqueue=True`` sink. Redaction matches shapes, not
30meaning: a secret in another form passes through, such as an arbitrary
31application-specific ``name: value`` pair with a colon, or a bare password
32or token with no label and no known shape. Nothing is claimed about an
33exception class or object deliberately written to defeat the redactor:
34metaclass, descriptor, ``__class__`` or ``__dict__`` hooks, a ``__str__``
35with side effects on other exceptions, a class registered in
36:mod:`builtins` under its own name, and the like. Such code can leak a
37secret in many other ways anyway. The checks below that reject such
38classes (rebuilding the exception from its checked text) are defence in
39depth, not a complete barrier; where a docstring says a check or a step
40"runs no code", that holds for exception classes and objects that do not
41hook those mechanisms.
42"""
44import builtins
45import collections.abc
46import contextlib
47import contextvars
48import copyreg
49import bisect
50import dataclasses
51import functools
52import itertools
53import operator
54import re
55import sys
56import types
57from importlib.machinery import ModuleSpec
58from multiprocessing.reduction import ForkingPickler
59from typing import Any, Callable, Iterable, NamedTuple, Optional, Union
61from .data_sanitizer import (
62 DataSanitizer,
63 REDACTION_TEXT,
64 _EXEMPT_QUALIFIER_LEAF,
65)
68# Strip C0/C1 control characters and dangerous Unicode format characters,
69# but preserve visible Unicode (accented, CJK, emoji, etc.)
70_UNSAFE_CHAR_RE = re.compile(
71 r"[\x00-\x1f\x7f-\x9f" # C0/C1 control chars
72 r"\u061c" # Arabic letter mark
73 r"\u200b-\u200f" # Zero-width chars + LTR/RTL marks
74 r"\u202a-\u202e" # Embedding/override (incl. RLO)
75 r"\u2028\u2029" # Line/paragraph separators — forced breaks in rendered
76 # HTML per CSS Text, so a log line carrying one can forge what looks
77 # like a separate entry even though re and str.splitlines() ignore them
78 r"\u2060-\u2064" # Word joiner + math invisible operators
79 r"\u2066-\u2069" # Isolate chars
80 r"\u206a-\u206f" # Digit shape controls
81 r"\ufeff" # BOM / zero-width no-break space
82 r"]"
83)
85# Default minimum length for a value to be considered a redactable secret.
86# Values shorter than this are skipped because a literal ``str.replace`` on
87# a short string would produce false positives in normal message content
88# (e.g. redacting the 3-char string ``key`` would scrub the word "key"
89# everywhere it appears).
90_MIN_SECRET_LENGTH = 8
92# Replacement token written in place of any redacted secret.
93_REDACTION_TOKEN = "***REDACTED***" # noqa: S105 # gitleaks:allow
96def strip_control_chars(value: str) -> str:
97 """Remove control and format characters from *value*, preserving visible Unicode."""
98 return _UNSAFE_CHAR_RE.sub("", value)
101def sanitize_log_record(record) -> None:
102 """loguru patcher stripping control characters from a record's message.
104 Control-character strip only. The web and MCP processes both install
105 :func:`redact_log_record` instead, which does this strip as part of its
106 message redaction and also redacts credentials.
107 """
108 record["message"] = strip_control_chars(record["message"])
111def sanitize_for_log(value: str, max_length: int = 50) -> str:
112 """Return a log-safe version of *value*.
114 * Control and format characters are stripped; valid Unicode is preserved.
115 * The result is truncated to *max_length* characters.
116 """
117 cleaned = strip_control_chars(value)
118 if len(cleaned) > max_length:
119 cleaned = (
120 cleaned[: max_length - 3] + "..."
121 if max_length > 3
122 else cleaned[:max_length]
123 )
124 return cleaned
127def redact_secrets(
128 message: str,
129 *secrets: Optional[str],
130 min_length: int = _MIN_SECRET_LENGTH,
131 replacement: str = _REDACTION_TOKEN,
132) -> str:
133 """Replace each occurrence of any *secret* in *message* with *replacement*.
135 Use this before writing a string to a log sink, returning it in an
136 error response, or persisting it \u2014 when the string may have been
137 constructed from upstream exception messages, URLs, or other
138 sources that could contain a value the caller already knows is
139 sensitive.
141 Each *secret* is matched as a literal substring (``str.replace``).
142 The function does not normalize encodings: if a secret appears
143 URL-encoded or otherwise transformed in *message*, the transformed
144 form is NOT redacted unless the caller also passes that
145 transformed form.
147 When multiple secrets are passed, they are applied in descending
148 length order so a shorter secret that happens to be a substring of
149 a longer one cannot consume part of the longer match. Example:
150 given secrets ``"abc12345"`` and ``"sk-abc12345"``, the longer one
151 is replaced first.
153 Args:
154 message: The string to scrub (a ``str`` subclass is scrubbed, and
155 returned, as a plain ``str``). Returned unchanged if it is
156 empty or a falsy non-string.
157 *secrets: Zero or more candidate secret values. ``None`` and
158 values shorter than *min_length* are silently skipped \u2014 the
159 caller is responsible for noticing missing config.
160 min_length: Minimum secret length to redact. Values shorter than
161 this are skipped to avoid corrupting normal message content
162 (a 1- or 2-character secret would match too aggressively).
163 Defaults to 8. Real API keys and session tokens are
164 typically 16+ characters.
165 replacement: String written in place of each redacted secret.
166 Defaults to ``"***REDACTED***"``.
168 Returns:
169 *message* with every occurrence of each qualifying secret
170 replaced.
172 See ``tests/security/test_log_sanitizer.py::TestRedactSecrets`` for
173 worked examples (doctest examples are constrained because the
174 repository's gitleaks rule flags token-shaped literals in
175 docstrings; ``.gitleaks.toml`` exempts only one audited historical
176 example value in this file, while other credential literals and
177 token-shaped values remain scanned).
178 """
179 message = _plain_text(message)
180 if not message:
181 return message
182 # Longest-first prevents a shorter overlapping secret from
183 # truncating a longer one once the replacement token is in place.
184 ordered = sorted(
185 (s for s in secrets if s and len(s) >= min_length),
186 key=len,
187 reverse=True,
188 )
189 for secret in ordered:
190 message = message.replace(secret, replacement)
191 return message
194# Pre-compiled regex patterns for common credential formats found in HTTP
195# library exception messages. Used by sanitize_error_message().
196#
197# Order matters: the URL-credentials pattern must run BEFORE the URL-param
198# pattern. Otherwise an input like ``?api-key=https://user:pass@host`` gets
199# its ``https`` consumed by the param replacement, the credentials pattern
200# no longer matches, and ``user:pass`` leaks.
201_CREDENTIAL_PATTERNS: list[tuple[re.Pattern[str], str]] = [
202 # Bearer tokens
203 (re.compile(r"Bearer\s+[A-Za-z0-9\-._~+/]+=*"), "Bearer [REDACTED]"),
204 # Authorization header WITH an explicit scheme. The scheme word is a
205 # strong anchor that rules out prose, so redact the credential on length
206 # alone (>=8 chars) regardless of its shape — this catches even an
207 # all-alphabetic Basic/Digest value. The scheme is preserved for
208 # debuggability. (A short prose word after a scheme is rare and would
209 # only be over-redacted, never leaked.)
210 (
211 re.compile(
212 r"(?i)(authorization\s*[:=]\s*)"
213 r"(basic|bearer|digest|negotiate|apikey|token)\s+"
214 r"[A-Za-z0-9\-._~+/]{8,}=*"
215 ),
216 r"\1\2 [REDACTED]",
217 ),
218 # Authorization header WITHOUT a scheme — here the value could be prose
219 # ("Authorization: required"), so require a *token-shaped* value (>=8
220 # chars containing a digit/+///=/_) to catch a raw token while leaving
221 # all-alphabetic prose intact.
222 (
223 re.compile(
224 r"(?i)(authorization\s*[:=]\s*)"
225 r"(?=[A-Za-z0-9\-._~+/]*[0-9+/=_])"
226 r"[A-Za-z0-9\-._~+/]{8,}=*"
227 ),
228 r"\1[REDACTED]",
229 ),
230 # x-api-key header — the label is a strong anchor (it doesn't appear in
231 # ordinary prose), so redact any sufficiently long value (>=16 chars)
232 # regardless of shape. Short values like "invalid"/"missing" stay intact.
233 (
234 re.compile(r"(?i)(x-api-key\s*[:=]\s*)[A-Za-z0-9\-._~+/]{16,}=*"),
235 r"\1[REDACTED]",
236 ),
237 # URL credentials (user:pass@host) in ANY URL scheme. Userinfo in a URL is
238 # always a credential, so this is not restricted to http(s): it also covers
239 # URL-form database connection strings — including SQLAlchemy's
240 # ``dialect+driver`` form (``postgresql+psycopg2://``, ``mysql+pymysql://``,
241 # ``mongodb+srv://``) and password-only DSNs (``redis://:pass@host``) —
242 # which are the most common credential-bearing strings in a raw DB/driver
243 # exception message. (Key=value DSNs like pyodbc's ``Server=...;Pwd=...``
244 # have no ``://`` and are out of scope for a userinfo regex.)
245 #
246 # The scheme is matched case-insensitively (URL schemes are case-insensitive
247 # per RFC 3986). NOTE: do NOT re-add a leading ``\b`` here — the scheme's
248 # first char is a word char, so ``\b`` fails to anchor when the URL is glued
249 # to a preceding word char (``Xhttps://user:pass@``) and the credential then
250 # leaks. The ``://`` literal plus the leading-letter requirement are already
251 # strong anchors against prose. The trailing ``@`` is required and ``/`` is
252 # excluded from the userinfo, so a credential-less DSN with a port
253 # (``postgresql://host:5432/db``) does not match.
254 (
255 re.compile(
256 r"([A-Za-z][A-Za-z0-9+.\-]{1,31}://)([^:\s/@]*):([^@\s/]+)@"
257 ),
258 r"\1[REDACTED]:[REDACTED]@",
259 ),
260 # Credential-bearing URL query parameters (?api_key=..., &access_token=...).
261 # Specific multi-word names precede the short catch-alls so the full
262 # parameter name is matched (e.g. ``secret_key`` not just ``secret``).
263 (
264 re.compile(
265 r"(?i)([?&])("
266 r"api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|"
267 r"auth[_-]?token|session[_-]?token|secret[_-]?key|bearer[_-]?token|"
268 r"subscription[_-]?key|client[_-]?secret|api[_-]?secret|"
269 r"app[_-]?secret|private[_-]?key|"
270 r"key|token|secret|password|passwd|pwd"
271 r")=([^&\s#]+)"
272 ),
273 r"\1\2=[REDACTED]",
274 ),
275 # Common API key prefixes (sk-*, pk-*) — includes hyphens for modern
276 # formats like sk-proj-... and sk-ant-api03-...
277 (re.compile(r"\b(sk-[A-Za-z0-9\-]{20,})\b"), "[REDACTED_KEY]"),
278 (re.compile(r"\b(pk-[A-Za-z0-9\-]{20,})\b"), "[REDACTED_KEY]"),
279 # Google API keys (AIza...) — match generously to cover length variants
280 # while the 20-char floor avoids short false positives.
281 (re.compile(r"\bAIza[0-9A-Za-z\-_]{20,}\b"), "[REDACTED_KEY]"),
282 # Distinctive provider token prefixes. These mirror the canonical,
283 # actively-maintained gitleaks ruleset (https://github.com/gitleaks/
284 # gitleaks, config/gitleaks.toml) — refresh from there when new token
285 # formats appear. They are prefix-anchored (very low false-positive risk
286 # in prose); the dual-scrub redact_secrets(known_literal) path remains the
287 # backstop for arbitrary/unknown secret shapes. See
288 # docs/developing/credential-scrubbing.md for the maintenance process.
289 # GitHub tokens: ghp_/gho_/ghu_/ghs_/ghr_ + fine-grained PATs.
290 (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"), "[REDACTED_KEY]"),
291 (re.compile(r"\bgithub_pat_[A-Za-z0-9_]{30,}\b"), "[REDACTED_KEY]"),
292 # AWS access key IDs (AKIA/ASIA/ABIA/ACCA/A3T...).
293 # Accepted false positive (over-redaction is the safe failure): a
294 # contiguous 20-char all-caps word starting with one of these prefixes
295 # is redacted. Mirrors gitleaks exactly.
296 (
297 re.compile(r"\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b"),
298 "[REDACTED_KEY]",
299 ),
300 # Slack tokens: xox[baeprs]-* (bot/user/app/refresh/config-refresh/...)
301 # and app-level xapp-* tokens. The ``xapp``/``xox`` prefixes are generic
302 # enough to appear in hyphenated identifiers, so (unlike the distinctive
303 # GitHub/AWS prefixes) we additionally require the long numeric workspace
304 # ID that every real Slack token carries (gitleaks expects ``[0-9]{10,13}``
305 # segments). This keeps real tokens redacted while leaving prose such as
306 # ``xapp-release-notes-2026`` intact.
307 #
308 # Linear-time form (these patterns run on every log record through
309 # redact_log_message): the naive ``\bxox[baeprs]-(?=[A-Za-z0-9-]*...)``
310 # lets EVERY ``xoxb-`` inside one ``[A-Za-z0-9-]`` run start an attempt
311 # whose lookahead rescans the rest of the run, so ``"xoxb-" * n`` is
312 # O(n^2) (measured: 10 KB 0.3 s, 20 KB 1.2 s). Every start inside the
313 # same run sees a suffix of what the run's leftmost ``\bxox`` start sees
314 # (same run end, same later digits, fewer characters), so if any start
315 # in a run matches, the leftmost one does. Attempts therefore begin only
316 # at a run start (the lookbehind) and commit, atomically, to the first
317 # ``\bxox*-`` in the run; the run prefix before it is captured and kept.
318 # Same matches as the naive form, one scan per run.
319 (
320 re.compile(
321 r"(?<![A-Za-z0-9-])(?>([A-Za-z0-9-]*?)\bxox[baeprs]-)"
322 r"(?=[A-Za-z0-9-]*[0-9]{9,})[A-Za-z0-9-]{10,}\b"
323 ),
324 r"\1[REDACTED_KEY]",
325 ),
326 (
327 re.compile(
328 r"(?<![A-Za-z0-9-])(?>([A-Za-z0-9-]*?)\bxapp-)"
329 r"(?=[A-Za-z0-9-]*[0-9]{9,})[A-Za-z0-9-]{10,}\b"
330 ),
331 r"\1[REDACTED_KEY]",
332 ),
333 # Google OAuth access tokens (ya29...).
334 (re.compile(r"\bya29\.[A-Za-z0-9_\-]{20,}"), "[REDACTED_KEY]"),
335 # JSON Web Tokens (three base64url segments). The two literal dots make
336 # this distinctive enough to avoid prose false positives. Accepted FP:
337 # a 3-part dotted identifier whose segments start with "eyJ" and are
338 # >=8 base64url chars (e.g. "eyJsonParser.eyJsonReader.eyJsonX") is
339 # redacted — over-redaction, not a leak. ``/`` is intentionally omitted
340 # (RFC 7515 JWTs are base64url); ``Bearer``/``Authorization`` paths
341 # already catch standard-base64 JWTs.
342 #
343 # Linear-time form, for the reason given on the Slack patterns above:
344 # ``.`` is outside the segment class, so a segment always runs to the end
345 # of its ``[A-Za-z0-9_+-]`` run, and the naive ``\beyJ...`` retried (and
346 # rescanned) at every ``eyJ`` inside one run — ``"eyJ-" * n`` was O(n^2)
347 # (320 KB: ~60 s). Only the run's leftmost ``\beyJ`` can matter (it
348 # has the longest first segment and the same run end), so attempts
349 # start at run starts and commit to it; the segments are possessive
350 # because backtracking inside a run can never reach a ``.``.
351 (
352 re.compile(
353 r"(?<![A-Za-z0-9_+\-])(?>([A-Za-z0-9_+\-]*?)\beyJ)"
354 r"[A-Za-z0-9_+\-]{8,}+\.[A-Za-z0-9_+\-]{8,}+\.[A-Za-z0-9_+\-]+"
355 ),
356 r"\1[REDACTED_KEY]",
357 ),
358]
360# Query string of any URL appearing in a message bound for logs/DB, e.g. the
361# "for url: https://host/path?..." suffix `requests.Response.raise_for_status`
362# appends to HTTPError. Only credential-*shaped* query params are redacted by
363# _CREDENTIAL_PATTERNS above; an arbitrary parameter (a search engine's `term=`,
364# an `id=`, a filter) carries no credential shape yet may still be user-supplied
365# text (e.g. the user's raw search query) that should not reach a log sink.
366# Scoped to scrub_error() only (see below) — NOT part of sanitize_error_message,
367# since that function also backs sanitize_error_for_client() — and, through it,
368# sanitize_error_for_agent(), which is that helper at a 500-char cap — where
369# preserving a failed request's own query string is useful for the caller
370# debugging it.
371#
372# Scheme: RFC 3986 §3.1 is one ALPHA followed by zero or more
373# ALPHA/DIGIT/+/-/. — the *lower* bound must stay 0 so a valid
374# single-character scheme (``x://...``) still matches, but the
375# *continuation* is capped at 31 chars (matches the sibling URL-credential
376# pattern above). An unbounded ``*`` here is a measured quadratic ReDoS:
377# on a message containing a long ALPHA/DIGIT/+/-/. run with no following
378# "://" (e.g. a stringified response body reaching str(error)), the engine
379# retries the "://" match at every run offset, backtracking the whole
380# remaining run each time — O(n^2). Verified locally: unbounded, a 40k-char
381# run takes ~1.5s (quadratic: ~4x per input doubling); bounded, the same
382# input is back to microseconds. See test_scrub_error_is_linear_time_on_a_
383# long_scheme_like_run.
384#
385# URL body (authority + path, up to the "?"): excludes whitespace, "?" and
386# "#" as before, plus the double quote and angle brackets, which are never
387# valid unencoded inside a URL (an HTTP client percent-encodes them), and is
388# capped at 4096 chars. Both are needed for the same reason the scheme
389# continuation is bounded: without them, ``[^\s?#]+`` followed by ``\?`` is a
390# second quadratic path — on a whitespace-free run dense in "://" (a
391# minified JSON list of URLs, a stringified response body) the engine
392# consumes to the end of the run at every "://" and backtracks the whole
393# remainder looking for a "?" that never comes. Measured on the uncapped
394# body: ~1s at 36k chars, ~4s at 72k, ~16s at 144k (4x per doubling), against
395# milliseconds before the query pass existed. Excluding the quote makes the
396# common JSON shape short per URL, and the cap bounds every other shape. A
397# URL whose scheme+authority+path exceeds 4096 chars has its query left
398# alone — no engine in this codebase builds one, and over-length is not a
399# leak vector the way a redaction gap is. See
400# test_scrub_error_is_linear_time_on_a_dense_url_run.
401#
402# Query stop-set: whitespace plus the wrappers a URL appears inside in
403# prose/logs (a markdown link's ``)``, a quote) so the redaction doesn't
404# consume past the URL's actual end and corrupt the surrounding message
405# (e.g. eating a markdown link's closing paren, or swallowing a second,
406# unspaced adjacent URL's own leading scheme once the first URL's wrapper is
407# reached). ``]`` and ``}`` are deliberately NOT in the stop-set: this pass
408# runs after sanitize_error_message, whose credential-shaped redaction emits
409# ``[REDACTED]`` / ``[REDACTED_KEY]`` in place of a value, and a stop-set
410# containing ``]`` halted the strip at that marker — so every parameter
411# AFTER a credential-shaped one (Google PSE's ``key=...&cx=...&q=<query>``,
412# ScaleSERP's ``api_key=...&q=<query>``) survived into the log while looking
413# scrubbed. A URL inside square brackets is now over-redacted up to the
414# closing bracket, which is the safe direction. Requires >=1 char (``+``,
415# not ``*``): a bare trailing "?" with nothing after it is not a query at
416# all, so a sentence-final "...page? yes" must not be rewritten into
417# "...page?<redacted> yes", which would imply a query existed.
418_URL_QUERY_STRING_RE = re.compile(
419 r"([A-Za-z][A-Za-z0-9+.\-]{0,31}://[^\s?#\"<>]{1,4096})\?([^\s)\"']+)"
420)
423_URL_QUERY_MARKER = "<redacted>"
424_URL_QUERY_WRAPPERS = {"<": ">", "`": "`"}
427def _restore_query_wrapper(match: "re.Match[str]") -> str:
428 """Keep only a final Markdown wrapper character, never query content."""
429 url, query = match.group(1), match.group(2)
430 opener = match.string[match.start() - 1] if match.start() else ""
431 closer = _URL_QUERY_WRAPPERS.get(opener)
432 if (
433 closer is not None
434 and len(query) > 1
435 and query.endswith(closer)
436 and query != _URL_QUERY_MARKER
437 ):
438 return f"{url}?{_URL_QUERY_MARKER}{closer}"
439 return f"{url}?{_URL_QUERY_MARKER}"
442def _query_needs_credential_guards(message: str, glued: str) -> bool:
443 """Whether query removal must preserve later credential readings."""
444 return "=" in message and (
445 _has_name_assignment_hint(message) or _has_name_assignment_hint(glued)
446 )
449def _redact_url_query_strings(message: str) -> str:
450 """Replace the query component of any URL in *message* with a fixed
451 marker, preserving the scheme/host/path so the endpoint stays legible.
453 A quoted value after a query assignment's ``=`` belongs to that query.
454 The regex stops at its opening quote; consume the whole value before
455 replacing the query, or a zero-width separator before the assignment
456 could leave the quoted secret visible without its credential name.
457 """
458 if "?" not in message or "://" not in message:
459 return message
460 queries = list(_URL_QUERY_STRING_RE.finditer(message))
461 if not queries:
462 return message
463 guards: list[list[tuple[int, int, int]]] = []
464 glued_query_ends: dict[int, int] = {}
465 glued = strip_control_chars(message)
466 split_offsets = _glued_offsets(message) if glued != message else None
467 if _query_needs_credential_guards(message, glued):
468 if glued == message:
469 to_text: Callable[[int], int] = int
470 else:
471 glued_starts, text_starts = _glued_offsets(message)
473 def to_text(offset: int) -> int:
474 run = bisect.bisect_right(glued_starts, offset) - 1
475 return text_starts[run] + offset - glued_starts[run]
477 guards = _log_patcher_spans(message, glued, to_text)
478 if glued != message:
479 glued_query_ends = {
480 to_text(query.start(2) - 1): to_text(query.end() - 1) + 1
481 for query in _URL_QUERY_STRING_RE.finditer(glued)
482 }
484 pieces: list[str] = []
485 done = 0
486 for match in queries:
487 if match.start() < done:
488 continue
489 query_end = match.end()
490 end = _cover_query_cut_credentials(
491 match.start(2) - 1, query_end, guards, query_end
492 )
493 if end > query_end:
494 # A name split by a control can sit inside a longer query in
495 # the stripped view. Consume that whole query so the suffix
496 # cannot be mistaken for URL userinfo on the next scrub round.
497 end = _cover_query_cut_credentials(
498 match.start(2) - 1,
499 max(end, glued_query_ends.get(match.start(2) - 1, end)),
500 guards,
501 query_end,
502 )
503 if (
504 end == match.end()
505 and match.group(2).endswith("=")
506 and end < len(message)
507 ):
508 quote = message[end]
509 if quote in ('"', "'"): 509 ↛ 526line 509 didn't jump to line 526 because the condition on line 509 was always true
510 end += 1
511 while end < len(message):
512 if message[end] == "\\":
513 end += 2
514 elif message[end] == quote:
515 end += 1
516 break
517 else:
518 end += 1
519 end = min(end, len(message))
520 # Extending through a quoted query value can consume the
521 # name of a later credential. Include its complete value
522 # so the log patcher is not left an unlabelled suffix.
523 end = _cover_query_cut_credentials(
524 match.start(2) - 1, end, guards, query_end
525 )
526 if split_offsets is not None:
527 while True:
528 extended = _cover_query_cut_credentials(
529 match.start(2) - 1,
530 _extend_over_split_credential(
531 glued,
532 *split_offsets,
533 match.start(2) - 1,
534 end,
535 ),
536 guards,
537 query_end,
538 )
539 if extended == end:
540 break
541 end = extended
542 pieces.append(message[done : match.start()])
543 pieces.append(_restore_query_wrapper(match))
544 if end > query_end:
545 pieces.extend(
546 char
547 for char in message[query_end:end]
548 if _UNSAFE_CHAR_RE.fullmatch(char)
549 )
550 done = end
551 if not pieces: 551 ↛ 552line 551 didn't jump to line 552 because the condition on line 551 was never true
552 return message
553 pieces.append(message[done:])
554 return "".join(pieces)
557_MAX_SCRUB_ROUNDS = 5
560def _scrub_to_fixed_point(scrub_once: Callable[[str], str], text: str) -> str:
561 """Repeat redaction until stable, with a bound on adversarial input."""
562 for _ in range(_MAX_SCRUB_ROUNDS): 562 ↛ 567line 562 didn't jump to line 567 because the loop on line 562 didn't complete
563 scrubbed = scrub_once(text)
564 if scrubbed == text:
565 break
566 text = scrubbed
567 return text
570def sanitize_error_message(message: str) -> str:
571 """Remove or mask API keys, tokens, and secrets from *message* using
572 pattern matching for common credential formats.
574 Use this as a first scrub pass on exception messages before logging,
575 followed by :func:`redact_secrets` with known literal values (the
576 "dual-scrub" pattern).
578 Handles:
579 * Bearer tokens (``Bearer sk-...``)
580 * ``Authorization:`` (any scheme) and ``x-api-key:`` headers
581 * URL query parameters (``?api_key=``, ``?access_token=``,
582 ``?refresh_token=``, ``?subscription-key=``, ``?secret=``, ...)
583 * URL-embedded credentials (``https://user:pass@host``)
584 * Well-known token prefixes — ``sk-``/``pk-``, Google ``AIza``/``ya29.``,
585 GitHub ``ghp_``/``github_pat_``, AWS ``AKIA``/``ASIA``, Slack ``xox*-``,
586 and JWTs (``eyJ….….…``). See ``docs/developing/credential-scrubbing.md``.
587 """
588 message = _plain_text(message)
589 return _scrub_to_fixed_point(_sanitize_error_message_once, message)
592def _sanitize_error_message_once(message: str) -> str:
593 if not message:
594 return message
595 for pattern, replacement in _CREDENTIAL_PATTERNS:
596 message = pattern.sub(replacement, message)
597 return message
600# A match found in the stripped view: start, end, replacement, and whether
601# the replacement is a fixed string holding none of the matched text.
602_GluedMatch = tuple[int, int, str, bool]
605def _glued_offsets(text: str) -> tuple[list[int], list[int]]:
606 """Map the text :func:`strip_control_chars` leaves back onto *text*.
608 Returns the start of each run of kept characters in the stripped text
609 and in *text*, in step, so a stripped offset ``i`` in run ``k`` is
610 ``text_starts[k] + i - glued_starts[k]`` in *text*.
611 """
612 glued_starts: list[int] = []
613 text_starts: list[int] = []
614 glued = 0
615 pos = 0
616 for unsafe in _UNSAFE_CHAR_RE.finditer(text):
617 if unsafe.start() > pos:
618 glued_starts.append(glued)
619 text_starts.append(pos)
620 glued += unsafe.start() - pos
621 pos = unsafe.end()
622 if pos < len(text):
623 glued_starts.append(glued)
624 text_starts.append(pos)
625 return glued_starts, text_starts
628def _log_patcher_spans(
629 text: str, glued: str, to_text: Callable[[int], int]
630) -> list[list[tuple[int, int, int]]]:
631 """Credentials in *text* that the log patcher's bare passes redact.
633 Every pass in :data:`_LOG_PATCHER_BARE_PASSES`, the list the patcher
634 itself applies (``name=value`` assignments and ``?name=value``
635 parameters), found both with the original separators and in the
636 stripped view (mapped back), as ``(start, value_start, end)``: one
637 sorted, non-overlapping list per pass and view.
638 """
639 lists = []
640 for pattern, _, name_group, value_after, _ in _LOG_PATCHER_BARE_PASSES:
642 def span(
643 m: "re.Match[str]",
644 name: int = name_group,
645 value: int = value_after,
646 ) -> tuple[int, int, int]:
647 return m.start(name), m.end(value), m.end()
649 lists.append([span(m) for m in pattern.finditer(text)])
650 lists.append(
651 [
652 (to_text(start), to_text(value), to_text(end - 1) + 1)
653 for start, value, end in map(span, pattern.finditer(glued))
654 ]
655 )
656 # Later passes can read a value differently after an inner name
657 # has been redacted. Guard those actual readings as well as the
658 # direct matches above; removing the outer name must also remove
659 # every value that any of the log pipelines associates with it.
660 for spans in _pipeline_credential_spans(text):
661 lists.extend(([], spans))
662 return lists
665def _fit_around_credentials(
666 start: int, end: int, guards: list[list[tuple[int, int, int]]]
667) -> Optional[tuple[int, int, bool]]:
668 """Fit the span ``[start, end)`` so the log patcher can still redact
669 every credential it touches.
671 A guard is a credential the patcher redacts (see
672 :func:`_log_patcher_spans`); the patcher needs its name to find its
673 value. A span that covers the whole value, or starts inside the value,
674 leaves nothing of the value visible without its name and is kept. A
675 span that starts before the guard and ends inside it is cut back to
676 end where the guard starts. A span that starts in the name and ends
677 before the value does is dropped (``None``). Returns the new span and
678 whether it changed.
679 """
680 lo, hi = start, end
681 changed = True
682 while changed: # cutting a span can uncover another partial overlap
683 changed = False
684 for spans in guards:
685 # Non-overlapping and sorted, so the ends are sorted too.
686 i = bisect.bisect_right(spans, lo, key=lambda span: span[2])
687 while i < len(spans) and spans[i][0] < hi:
688 g_start, value_start, g_end = spans[i]
689 i += 1
690 if lo >= value_start or hi >= g_end:
691 continue
692 if lo >= g_start:
693 return None
694 hi = g_start
695 changed = True
696 return lo, hi, (lo, hi) != (start, end)
699def _cover_cut_credentials(
700 start: int, end: int, guards: list[list[tuple[int, int, int]]]
701) -> int:
702 """End of the span ``[start, end)``, extended to the end of every
703 credential in *guards* (see :func:`_log_patcher_spans`) whose name it
704 takes without the whole of its value, so the patcher is never left that
705 value without its name."""
706 changed = True
707 while changed:
708 changed = False
709 for spans in guards:
710 i = bisect.bisect_right(spans, start, key=lambda span: span[2])
711 while i < len(spans) and spans[i][0] < end:
712 g_start, value_start, g_end = spans[i]
713 i += 1
714 if start < value_start and end < g_end:
715 end = g_end
716 changed = True
717 return end
720def _cover_query_cut_credentials(
721 start: int,
722 end: int,
723 guards: list[list[tuple[int, int, int]]],
724 query_end: int,
725) -> int:
726 """Cover credentials cut by a query, retaining intact later assignments.
728 A glued-view match can begin in the final query character and extend
729 across a line break into a complete assignment. If the original-view
730 patcher already covers that whole assignment after *query_end*, leave
731 the line and its credential untouched. Span lists alternate original
732 and glued views, so only overlapping spans need to be checked.
733 """
735 def preserved_in_original(value_start: int, value_end: int) -> bool:
736 for originals in guards[::2]:
737 j = bisect.bisect_left(
738 originals, value_start, key=lambda span: span[1]
739 )
740 if (
741 j < len(originals)
742 and originals[j][0] >= query_end
743 and originals[j][1] == value_start
744 and originals[j][2] >= value_end
745 ):
746 return True
747 return False
749 changed = True
750 while changed:
751 changed = False
752 for view, spans in enumerate(guards):
753 i = bisect.bisect_right(spans, start, key=lambda span: span[2])
754 while i < len(spans) and spans[i][0] < end:
755 g_start, value_start, g_end = spans[i]
756 i += 1
757 if start >= value_start or end >= g_end:
758 continue
759 if view % 2 and preserved_in_original(value_start, g_end):
760 continue
761 end = g_end
762 changed = True
763 return end
766# What the log patcher's bare passes take as an unquoted value: the run of
767# non-whitespace after the ``=`` (a stop inside it only makes it shorter).
768_NON_SPACE_RUN_RE = re.compile(r"\S*")
771def _reveals_log_patcher_name(covered: str, replacement: str) -> bool:
772 """True when *replacement*, written in place of *covered*, shows the log
773 patcher a ``name=`` or ``?name=`` it does not see in *covered*.
775 The stripped view can join a name split by an invisible character
776 (``authorizatio<U+200B>n=...``), and a credential pattern's replacement
777 keeps it (``authorization=[REDACTED]``). The patcher then takes the run
778 of text after the replacement as that name's value, up to the next
779 whitespace, and that run can be the first half of another name split by
780 a line break (``'api_`` of ``'api_\\nkey=...``), which it would no
781 longer recognise. :func:`_splice_across_controls` therefore takes that
782 run into the redaction, and so the credential it starts.
783 """
784 return any(
785 bare.unanchored.search(replacement)
786 and not bare.unanchored.search(covered)
787 for bare in _LOG_PATCHER_BARE_PASSES
788 )
791# How many line breaks (or other stripped characters) at the end of a span
792# :func:`_extend_over_split_credential` looks at, and how far before the
793# first of them and past the span it searches at a time. Bounds its cost
794# on adversarial input.
795_MAX_PROBED_CROSSINGS = 64
796_PROBE_WINDOW = 128
799def _extend_over_split_credential(
800 glued: str,
801 glued_starts: list[int],
802 text_starts: list[int],
803 start: int,
804 end: int,
805) -> int:
806 """End of the span ``[start, end)`` of *text*, extended so it does not
807 take only the start of a credential that runs on past it.
809 In the stripped view a span can run on through a line break into the
810 next line and take the start of a credential there: a name, or a
811 ``Bearer``/``Authorization:`` anchor split by a stripped character
812 (``Beare\\rr``), whose value lies past the span. Without that start
813 the log patcher no longer recognises the value. So when a credential
814 pattern the patcher uses matches across the span's end (as a scan from
815 before the span finds it, or starting right after a stripped character
816 inside the span) and would still redact something, the span is extended
817 to the end of that match, and the value is redacted with it. *glued_starts* and *text_starts* are from :func:`_glued_offsets`.
818 """
819 probes = [
820 *_CREDENTIAL_PATTERNS,
821 *(
822 (bare.unanchored, bare.replacement)
823 for bare in _LOG_PATCHER_BARE_PASSES
824 ),
825 ]
826 anchored = {
827 bare.unanchored: bare.pattern for bare in _LOG_PATCHER_BARE_PASSES
828 }
830 def to_glued(offset: int) -> int:
831 """Stripped-view offset of the first kept character at or after
832 *offset* in *text*."""
833 run = bisect.bisect_right(text_starts, offset) - 1
834 if run < 0: 834 ↛ 835line 834 didn't jump to line 835 because the condition on line 834 was never true
835 return 0
836 run_len = (
837 glued_starts[run + 1] if run + 1 < len(glued_starts) else len(glued)
838 ) - glued_starts[run]
839 return glued_starts[run] + min(offset - text_starts[run], run_len)
841 def to_text_end(glued_end: int) -> int:
842 """Offset in *text* just after the kept character before
843 *glued_end*."""
844 run = bisect.bisect_right(glued_starts, glued_end - 1) - 1
845 return text_starts[run] + glued_end - glued_starts[run]
847 # Bound chains that repeatedly expose another credential. If the work
848 # limit is reached, redact the remaining tail instead of leaving a value
849 # behind after consuming its label.
850 extensions_left = 16
851 while True:
852 if extensions_left == 0:
853 return to_text_end(len(glued))
854 extensions_left -= 1
855 # Runs of kept text that begin inside the span follow a stripped
856 # character there.
857 first = bisect.bisect_right(text_starts, start)
858 last = bisect.bisect_left(text_starts, end)
859 runs = range(max(first, last - _MAX_PROBED_CROSSINGS), last)
860 if not runs:
861 return end
862 crossings = [glued_starts[run] for run in runs]
863 glued_end = to_glued(end)
864 window_end = min(len(glued), glued_end + _PROBE_WINDOW)
865 reach = glued_end
867 def runs_on(m: Optional["re.Match[str]"], template: str) -> int:
868 """End of *m* if it runs past the span and would still redact
869 something, else 0. A match cut off by the window is redone
870 without it, so the whole value is covered."""
871 if (
872 m is None
873 or m.end() <= glued_end
874 or m.expand(template) == m.group()
875 ):
876 return 0
877 if m.end() >= window_end:
878 whole = m.re.match(glued, m.start())
879 if whole is not None: 879 ↛ 881line 879 didn't jump to line 881 because the condition on line 879 was always true
880 return max(m.end(), whole.end())
881 return m.end()
883 window_start = max(0, crossings[0] - _PROBE_WINDOW)
884 for pattern, template in probes:
885 # A match as the patcher finds it, scanning from before the
886 # span, that straddles the span's end.
887 natural = anchored.get(pattern, pattern)
888 for m in natural.finditer(glued, window_start, window_end):
889 if m.start() >= glued_end:
890 break
891 reach = max(reach, runs_on(m, template))
892 if "://" in pattern.pattern:
893 continue # a URL scheme has no fixed start to anchor at
894 # A match starting right after a stripped character: the
895 # patcher may see it there once what precedes it is redacted.
896 for crossing in crossings:
897 m = pattern.match(glued, crossing, window_end)
898 reach = max(reach, runs_on(m, template))
899 if reach == glued_end:
900 return end
901 end = to_text_end(reach)
904def _splice_across_controls(
905 text: str, glued: str, matches: Iterable[_GluedMatch]
906) -> str:
907 """Apply redactions found in *glued* (``strip_control_chars(text)``)
908 to *text* itself.
910 Each match is mapped back to the span of *text* it covers, then fitted
911 around any credential the log patcher would redact (see
912 :func:`_fit_around_credentials`) and extended over any credential it
913 would otherwise cut into (see :func:`_extend_over_split_credential`),
914 and over the whole of any credential that extension takes the name of
915 (see :func:`_cover_cut_credentials`). A replacement that would show the
916 patcher a name it does not see in the span (a split name the stripped
917 view joined, see :func:`_reveals_log_patcher_name`) is extended over the
918 run of text the patcher would take as that name's value. These rules
919 keep the patcher from being left a value without its name in the
920 shapes they model; they are not a proof for every input. A span that
921 had to change gets a plain
922 ``[REDACTED]`` unless its replacement holds no matched text. The
923 whitespace control characters inside a replaced span (line breaks,
924 tabs, U+2028, ...) are written back after the replacement, so no two
925 lines are joined. A span whose stripped text already equals its
926 replacement is left alone.
927 """
928 glued_starts, text_starts = _glued_offsets(text)
930 def to_text(offset: int) -> int:
931 run = bisect.bisect_right(glued_starts, offset) - 1
932 return text_starts[run] + offset - glued_starts[run]
934 guards = _log_patcher_spans(text, glued, to_text)
935 pieces: list[str] = []
936 done = 0
937 for g_start, g_end, replacement, literal in matches:
938 if g_end <= g_start: 938 ↛ 939line 938 didn't jump to line 939 because the condition on line 938 was never true
939 continue
940 fitted = _fit_around_credentials(
941 to_text(g_start), to_text(g_end - 1) + 1, guards
942 )
943 if fitted is None:
944 continue
945 start, end, shrunk = fitted
946 if shrunk and not literal:
947 # A shape replacement retains its anchor (header name, URL
948 # scheme, ...). Cutting its match short and replacing that
949 # anchor with a plain marker would expose the original value.
950 # Keep the whole match; the extension below also conceals any
951 # overlapping assignment whose name this span would take.
952 end = to_text(g_end - 1) + 1
953 if start < done:
954 continue # inside a span an earlier match was extended over
955 if not (shrunk and not literal) and _reveals_log_patcher_name(
956 text[start:end], replacement
957 ):
958 run_end = _NON_SPACE_RUN_RE.match(text, end).end()
959 if run_end != end:
960 end, shrunk = run_end, True
961 while True:
962 # Extending can take the name of a credential the patcher sees
963 # in *text*; the span then takes its whole value as well.
964 extended = _cover_cut_credentials(
965 start,
966 _extend_over_split_credential(
967 glued, glued_starts, text_starts, start, end
968 ),
969 guards,
970 )
971 if extended == end:
972 break
973 end, shrunk = extended, True
974 if shrunk and not literal:
975 replacement = REDACTION_TEXT
976 covered = text[start:end]
977 if strip_control_chars(covered) == replacement:
978 continue
979 kept = "".join(
980 c for c in covered if c.isspace() and _UNSAFE_CHAR_RE.match(c)
981 )
982 pieces.append(text[done:start])
983 pieces.append(replacement + kept)
984 done = end
985 pieces.append(text[done:])
986 return "".join(pieces)
989# Longest text :func:`_scrub_across_controls` scans. On adversarial input
990# (a line break inside every ``Bearer``, ``Authorization:`` or URL) it costs
991# up to about 55 microseconds per character, against about 0.4 for the
992# plain passes, so at this bound about 0.2 s per round. Longer text gets the
993# plain passes only, as before that view existed.
994_GLUED_VIEW_MAX_CHARS = 4096
997def _scrub_across_controls(text: str, secrets: list) -> str:
998 """Re-run :func:`scrub_error`'s passes on *text* with control and
999 invisible format characters removed, applying what they find to
1000 *text* itself (see :func:`_splice_across_controls`).
1002 Text without such characters is returned as it is: the plain passes
1003 already saw it exactly as this view would. So is text longer than
1004 ``_GLUED_VIEW_MAX_CHARS``, which bounds the cost of this view; such text
1005 keeps only the plain passes, and the log patcher still strips control
1006 characters and redacts again before it is logged.
1007 """
1008 if len(text) > _GLUED_VIEW_MAX_CHARS or not _UNSAFE_CHAR_RE.search(text):
1009 return text
1010 glued = strip_control_chars(text)
1012 def apply(find: Callable[[str], Iterable[_GluedMatch]]) -> None:
1013 nonlocal text, glued
1014 matches = list(find(glued))
1015 if matches:
1016 spliced = _splice_across_controls(text, glued, matches)
1017 if spliced != text:
1018 text = spliced
1019 glued = strip_control_chars(text)
1021 for pattern, template in _CREDENTIAL_PATTERNS:
1022 apply(
1023 lambda glued, p=pattern, t=template: (
1024 (m.start(), m.end(), m.expand(t), "\\" not in t)
1025 for m in p.finditer(glued)
1026 )
1027 )
1028 for secret in sorted(
1029 (s for s in secrets if s and len(s) >= _MIN_SECRET_LENGTH),
1030 key=len,
1031 reverse=True,
1032 ):
1033 apply(lambda glued, s=secret: _literal_matches(glued, s))
1035 def query_matches(glued: str) -> Iterable[_GluedMatch]:
1036 for m in _URL_QUERY_STRING_RE.finditer(glued):
1037 # Only the query is replaced; the URL before it keeps its text.
1038 yield (
1039 m.start(2) - 1,
1040 m.end(),
1041 _restore_query_wrapper(m)[len(m.group(1)) :],
1042 True,
1043 )
1045 apply(query_matches)
1046 return text
1049def _literal_matches(glued: str, secret: str) -> Iterable[_GluedMatch]:
1050 """Non-overlapping occurrences of *secret*, as ``str.replace`` finds them."""
1051 pos = glued.find(secret)
1052 while pos != -1:
1053 yield (pos, pos + len(secret), _REDACTION_TOKEN, True)
1054 pos = glued.find(secret, pos + len(secret))
1057def scrub_error(error: Union[BaseException, str], *secrets: Any) -> str:
1058 """Return a log/DB-safe rendering of *error* (the "triple-scrub").
1060 Composes the scrub passes every catch site needs:
1061 :func:`sanitize_error_message` (catches credential *shapes* — Bearer
1062 tokens, URL-embedded credentials, ``sk-``/``pk-`` keys), then
1063 :func:`redact_secrets` with the caller's known literal secret values,
1064 then a blanket strip of any URL's query string (``?...`` →
1065 ``?<redacted>``) so a non-credential-shaped parameter — e.g. a search
1066 engine's ``term=`` — cannot carry the caller's raw query text into
1067 logs via a library-formatted message such as
1068 ``requests``' ``"... for url: https://host/path?term=<query>"``. This
1069 last pass is specific to ``scrub_error``, not :func:`sanitize_error_message`
1070 (which also backs the client-facing :func:`sanitize_error_for_client` and,
1071 through it, the agent-facing :func:`sanitize_error_for_agent` — where
1072 preserving a failed request's own query string is useful for the caller
1073 debugging it).
1075 Use this at every catch site that logs or persists an exception so
1076 the passes can never drift apart per-site.
1077 ``BaseSearchEngine._scrub_error`` delegates here, resolving its
1078 engine's ``_secret_attrs`` into the *secrets* arguments.
1080 Query removal needs additional credential-boundary analysis when a
1081 URL query and a sensitive assignment occur together. Such text is cut
1082 at the log redactor's whitespace-safe 32 KiB bound after shape and
1083 known-literal redaction, before that expensive analysis; the omitted
1084 tail is replaced by a marker outside the fixed-point rounds. Other
1085 long diagnostics retain their full-text behavior.
1087 Defensive by design: this runs inside ``except`` blocks, so it must
1088 never raise. ``str(error)`` is guarded (a custom exception whose
1089 ``__str__`` raises won't crash the handler) and each secret is coerced
1090 to ``str`` (a misconfigured non-string secret, e.g. an int from
1091 settings, won't trip ``redact_secrets``' ``len()`` check).
1093 Args:
1094 error: An exception or a pre-built message string.
1095 *secrets: Known literal secret values to redact. ``None`` and
1096 falsy values are silently skipped.
1098 Returns:
1099 The scrubbed message, safe for production log sinks.
1100 """
1101 try:
1102 message = str(error)
1103 except Exception:
1104 message = "<unprintable {}>".format(type(error).__name__)
1105 # Coerce truthy non-str secrets to str; keep None/falsy as-is
1106 # (redact_secrets filters those out). Guarded per secret: a
1107 # pathological value whose __bool__/__str__ raises cannot be
1108 # literal-matched anyway, so it is skipped rather than allowed to
1109 # crash the except handler this runs in.
1110 safe_secrets = []
1111 for v in secrets:
1112 try:
1113 safe_secrets.append(v and str(v))
1114 except Exception:
1115 continue
1117 omitted = 0
1119 def scrub_once(text: str) -> str:
1120 nonlocal omitted
1121 text = redact_secrets(sanitize_error_message(text), *safe_secrets)
1122 if (
1123 len(text) > _LOG_REDACTION_MAX_CHARS
1124 and "?" in text
1125 and "://" in text
1126 and _query_needs_credential_guards(text, strip_control_chars(text))
1127 and _URL_QUERY_STRING_RE.search(text) is not None
1128 ):
1129 # Shape and known-literal redaction must precede the cut: a
1130 # known secret containing spaces could straddle its boundary.
1131 # Guarded query removal tracks every character through several
1132 # readings; limit that work before creating the origin arrays.
1133 # Ordinary long diagnostics keep their existing full-text path.
1134 head, _ = _truncate_for_log_redaction(text)
1135 omitted += len(text) - len(head)
1136 text = head
1137 text = _redact_url_query_strings(text)
1138 return _scrub_across_controls(text, safe_secrets)
1140 result = _scrub_to_fixed_point(scrub_once, message)
1141 if omitted:
1142 # Keep generated metadata outside subsequent redaction rounds.
1143 result += f" [... {omitted} characters omitted from log output]"
1144 return result
1147def sanitize_error_for_client(message: str, max_length: int = 200) -> str:
1148 """Make an exception-derived string safe to return to an HTTP client.
1150 Composes :func:`sanitize_error_message` (credential redaction) and
1151 :func:`sanitize_for_log` (control-char strip + length cap). Credential
1152 scrubbing runs FIRST, on the full untruncated string, so a secret near
1153 the ``max_length`` boundary cannot be split by truncation and slip past
1154 the regexes.
1156 Scope, because it is easy to over-trust: this removes credential
1157 *shapes*. It does NOT remove server filesystem paths, SQL text, provider
1158 endpoints or dependency internals. Use it on a message an author already
1159 knows to be safe apart from a possible embedded secret — not as a filter
1160 that makes an arbitrary exception safe to show.
1162 Where an exception can carry those other kinds of detail, do not pass its
1163 text through here at all. Two worked examples, and they are NOT the same
1164 technique:
1166 * ``web/routers/rag.py``'s ``_format_test_embedding_error`` withholds the
1167 text entirely — it uses the exception's module and class only to SELECT
1168 one of a fixed set of messages, and interpolates neither.
1169 * ``web/routers/zotero.py``'s ``_zotero_error_response`` forwards text,
1170 but only text that is already one of the package's author-written
1171 constants: ``client_safe_zotero_message`` looks ``str(exc)`` up in
1172 ``CLIENT_SAFE_ZOTERO_MESSAGES`` and returns the module constant it
1173 found (or the caller's own fallback literal), so the value in the
1174 response never comes from the exception object.
1176 Keep the detail server-side. ``logger.exception`` is the default, but it
1177 is not unconditional: loguru's ``diagnose`` renders every frame-local in
1178 an attached traceback, so in a handler whose frames hold credentials (an
1179 API key, the SQLCipher password) log without the traceback instead —
1180 ``logger.warning(scrub_error(exc, *known_secrets))``. See the Zotero sync
1181 handler in ``research_library/zotero/sync_service.py`` and the
1182 credential-frame handlers in ``web/queue/processor_v2.py``. Only the
1183 stderr sink can have ``diagnose`` on, and only behind two explicit
1184 opt-ins (``utilities/log_utils.py``), but that is exactly the
1185 configuration an operator debugging such a failure runs.
1186 """
1187 scrubbed = _scrub_to_fixed_point(
1188 lambda text: strip_control_chars(_sanitize_error_message_once(text)),
1189 message,
1190 )
1191 return sanitize_for_log(scrubbed, max_length=max_length)
1194# Larger cap than ``sanitize_error_for_client``'s HTTP-client default: these
1195# strings feed the agent's reasoning AND the ErrorReporter pattern map, where
1196# over-aggressive truncation drops the categorizable error signal. Credential
1197# scrubbing still runs first on the full untruncated string (#4633).
1198_AGENT_ERROR_MAX_LEN = 500
1201def sanitize_error_for_agent(message: str) -> str:
1202 """Scrub an exception-derived string that is bound for an LLM / agent.
1204 A preset over :func:`sanitize_error_for_client`: it removes credential
1205 *shapes* and control characters and caps the result at 500 characters.
1206 It delegates rather than re-composing those passes, so the agent path
1207 cannot drift away from the client path — the cap is the only
1208 difference between the two.
1210 Why 500 and not the client helper's 200-char default: a tool error
1211 feeds the model's reasoning loop and the ``ErrorReporter`` pattern map,
1212 and the classification signal often sits deep in a provider message (a
1213 rate-limit phrase such as ``"429 Too Many Requests"`` behind a long
1214 upstream prefix). A 200-char cap removes that signal before anything
1215 downstream can act on it.
1217 Scope, because it is easy to over-trust: this removes credential
1218 *shapes*. It does NOT remove server filesystem paths, SQL text,
1219 provider endpoints or dependency internals. Use it on a message the
1220 caller already knows to be safe apart from a possible embedded secret
1221 — not as a filter that makes an arbitrary exception safe to show.
1222 Where an exception can carry those other kinds of detail, do not pass
1223 its text through here at all: select one of a fixed set of messages
1224 from the exception's type, or forward only author-written constants,
1225 and keep the detail server-side in the logs.
1227 "Agent" is not a routing boundary. Agent-facing text does reach HTTP
1228 responses: a strategy's ``formatted_findings`` travels through
1229 ``api/research_functions.py`` to ``web/routers/api_v1.py``, whose
1230 boundary re-applies the same credential-shape pass. Choosing between
1231 this helper and :func:`sanitize_error_for_client` picks a cap; it does
1232 not decide where the string ends up.
1233 """
1234 return sanitize_error_for_client(message, max_length=_AGENT_ERROR_MAX_LEN)
1237#: Characters that delimit every secret-bearing part of a URL: path ``/``
1238#: (and ``\\``), query ``?``, fragment ``#``, userinfo ``@`` and path
1239#: parameters ``;``. A string with none of them can hold at most a host.
1240_URL_PART_DELIMITERS = frozenset("/\\?#@;")
1242#: Placeholder logged instead of a failure reason that may embed a URL.
1243FAILURE_REASON_WITHHELD = "<withheld: may embed a URL>"
1246def failure_reason_for_log(reason: Any) -> str:
1247 """Return a download failure reason in a form that is safe to log.
1249 The reason strings that ``DownloadService.download_resource`` and
1250 ``download_as_text`` return are usually fixed categories
1251 (``"egress_policy_denied:..."``, ``"No compatible downloader
1252 available"``), but their exception paths return
1253 ``sanitize_error_for_client(str(e))``. That removes credential
1254 *shapes* only: request and urllib3 exception texts keep the URL's path
1255 and query (``with url: /p?X-Amz-Signature=...``), so presigned tokens
1256 survive it. The exception path logs its own cause (exception type and
1257 redacted URL) where it is caught, so a log line further up needs only
1258 the category.
1260 Fails closed on shape: a reason containing any URL-part delimiter is
1261 replaced by :data:`FAILURE_REASON_WITHHELD`; anything else is returned
1262 unchanged. The test is on shape, not origin, so a fixed reason that
1263 happens to contain a delimiter (``"Unexpected content type: text/html
1264 - expected PDF"``) is withheld too; that loss of detail is accepted
1265 rather than trying to tell a fixed reason from exception text. The
1266 result is for logs only: callers keep passing the original reason to
1267 their response or database.
1268 """
1269 if reason is None:
1270 return "None"
1271 if not isinstance(reason, str):
1272 return f"<{type(reason).__name__}>"
1273 if any(char in _URL_PART_DELIMITERS for char in reason):
1274 return FAILURE_REASON_WITHHELD
1275 return reason
1278def sanitize_error_details(value: Any) -> Any:
1279 """Recursively redact credential *shapes* from the string leaves of a
1280 structured ``details`` value (``dict`` / ``list`` / ``tuple`` / dataclass).
1282 Intended for the ``details`` payload of an exception ``to_dict()`` that is
1283 serialized to a client (e.g. ``NewsAPIException`` / ``WebAPIException``).
1284 This is the *value-shape* counterpart to :class:`data_sanitizer.DataSanitizer`,
1285 which redacts by *key name*; use this one when the concern is a credential
1286 embedded anywhere in the text, regardless of the key it sits under.
1288 Behaviour by node type:
1290 * ``dict`` — recurse into values; ``str`` keys are also run through
1291 :func:`sanitize_error_message` (a credential used *as* a key would
1292 otherwise ship verbatim as a JSON key). A key whose redacted form is
1293 already taken (two URLs with different passwords, or a redacted key
1294 equal to a later literal key) gets a ``#2``/``#3`` suffix instead of
1295 overwriting that entry; the first key in iteration order keeps the
1296 unsuffixed name.
1297 * ``list`` / ``tuple`` — both rebuilt as a plain ``list``: the payload is
1298 about to be JSON-serialized (a tuple already becomes a JSON array), and
1299 rebuilding via ``type(value)(<generator>)`` would raise on a namedtuple /
1300 tuple subclass whose constructor is not ``(iterable) -> instance`` —
1301 inside a Flask error handler that degrades a structured error into a bare
1302 500.
1303 * **dataclass instance** — converted via ``dataclasses.asdict`` and recursed.
1304 Flask's default JSON provider serializes a dataclass (via ``asdict``), so
1305 without this a credential in a dataclass field would ship un-redacted.
1306 * ``str`` leaf — redacted via :func:`sanitize_error_message` (credential-shape
1307 redaction only — no length cap or control-char strip, so structured values
1308 such as IDs survive).
1309 * anything else (ints, bools, ``None``, and any other object) — passed
1310 through untouched. Note some passthrough types (``set``/``frozenset``,
1311 ``bytes``, a non-``str``-mixin ``Enum``) are not JSON-serializable, so a
1312 credential inside them fails *closed* at ``jsonify`` (generic 500) rather
1313 than leaking.
1315 Redaction is shape-based, so it never removes a value by key name — a benign
1316 ``{"query": "reset my password"}`` is unchanged. Containers are rebuilt fresh
1317 (the input is not mutated); passthrough leaf objects are returned by
1318 reference.
1319 """
1320 if isinstance(value, dict):
1321 out: dict = {}
1322 for k, v in value.items():
1323 safe_key = sanitize_error_message(k) if isinstance(k, str) else k
1324 out[_unique_log_key(out, safe_key)] = sanitize_error_details(v)
1325 return out
1326 if isinstance(value, (list, tuple)):
1327 return [sanitize_error_details(v) for v in value]
1328 # dataclass *instance* (not the class itself) — Flask serializes it via
1329 # asdict(), so recurse into its fields to redact any credential-shaped one.
1330 if dataclasses.is_dataclass(value) and not isinstance(value, type):
1331 return sanitize_error_details(dataclasses.asdict(value))
1332 if isinstance(value, str):
1333 return sanitize_error_message(value)
1334 return value
1337# ---------------------------------------------------------------------------
1338# Log-sink credential redaction (the backstop applied by the loguru patcher)
1339# ---------------------------------------------------------------------------
1340#
1341# ``utilities/log_utils.py`` installs :func:`redact_log_message` /
1342# :func:`redact_log_extra` as a global loguru *patcher* so every record is
1343# scrubbed once, before it reaches ANY sink — stderr, the encrypted per-user
1344# DB, the browser progress stream, and (when enabled) the persistent,
1345# unencrypted ``<LDR_DATA_DIR>/logs/*.log`` file.
1346#
1347# The sanctioned per-call-site pattern is ``logger.error(scrub_error(msg,
1348# secret))`` (see :func:`scrub_error`). This sink-level pass is the BACKSTOP
1349# for the recurring class of bug where a call site forgets it: a credential
1350# that rides the message as a bare ``field=value`` pair, or that is bound as
1351# a structured ``extra`` value under a key the name-based checks miss. Both
1352# the message string and the structured extras are scrubbed here.
1354# Settings redaction's default secret names, plus credentials that appear only
1355# in logs. Routing fields remain explicitly exempt below. This is a set of
1356# exact names. Message text also recognises environment-variable and dotted
1357# setting prefixes for these names (``OPENAI_API_KEY``, ``db_password``,
1358# ``llm.openai.api_key``); structured ``extra`` keys use the same suffix rule
1359# (see ``_is_sensitive_log_key``). The later quoted-key pass also handles
1360# JSON and Python-repr credential keys (see ``_QUOTED_KEY_PREFIX``).
1361LOG_SENSITIVE_KEYS: frozenset = frozenset(
1362 DataSanitizer.DEFAULT_SENSITIVE_KEYS
1363) | frozenset(
1364 {
1365 "passwd",
1366 "pwd",
1367 "user_password",
1368 "encryption_key",
1369 "sqlcipher_key",
1370 "derived_key",
1371 "salt",
1372 "authorization",
1373 "id_token",
1374 "subscription_key",
1375 }
1376)
1378# ``extra`` keys the log sinks read for routing / behaviour. They are never
1379# redacted, or the DB sink loses per-user attribution
1380# (``database_sink``/``_get_research_id`` read ``username``/``research_id``)
1381# and the frontend filter loses its ``policy_audit`` guard. None of these
1382# hold a secret.
1383_LOG_STRUCTURAL_EXTRA_KEYS: frozenset = frozenset(
1384 {
1385 "research_id",
1386 "username",
1387 "policy_audit",
1388 }
1389)
1391# Keep query and bare-assignment credential names in agreement, but retain
1392# their distinct value boundaries so harmless query parameters survive.
1393_LOG_SENSITIVE_NAME_PATTERN = "|".join(
1394 re.escape(name).replace("_", r"[_-]?")
1395 for name in sorted(LOG_SENSITIVE_KEYS, key=lambda name: (-len(name), name))
1396)
1397# A following assignment can use the same prefixed or dotted credential
1398# names as the log patcher. The settings flag remains exempt: its value is
1399# not a credential and must not be used to shorten a preceding redaction.
1400_STOP_SENSITIVE_NAME_PATTERN = "".join(
1401 (
1402 r"(?!(?:[a-z0-9]+[._])*requires[_-]api[_-]key\s*=)",
1403 r"(?:[a-z0-9]+[._])*(?:",
1404 _LOG_SENSITIVE_NAME_PATTERN,
1405 r")",
1406 )
1407)
1408_SENSITIVE_QUERY_RE = re.compile(
1409 r"(?i)([?&](?:" + _LOG_SENSITIVE_NAME_PATTERN + r")=)([^&\s#]+)"
1410)
1412# The name is a sensitive name, optionally preceded by any number of
1413# ``<alnum>_`` / ``<alnum>.`` segments: the same suffix rule ``extra`` keys
1414# get, applied to env-var (``OPENAI_API_KEY``), prefixed (``db_password``)
1415# and dotted setting (``llm.openai.api_key``) spellings. The whole name must
1416# start a word (not after ``\w``, ``.``, ``-``, or a query's ``?``/``&``),
1417# so ``mytoken=`` and ``x-token=`` stay readable, and it must END at the
1418# ``=``: ``max_tokens=`` is not ``_token``. After ``?``/``&``,
1419# ``_SENSITIVE_QUERY_RE`` redacts the exact names only: a prefixed name in
1420# a query string (``&google_api_key=x``) is not redacted, nor is any name
1421# after ``-`` (``--db_password=x``, ``--password=x``). The prefix
1422# segments cannot contain a separator, so each start position scans one
1423# identifier once -- linear, like the rest of the pass.
1424#
1425# Quoted values may contain spaces and escaped quotes. An unfinished quote
1426# consumes the remaining message. Unquoted values end at whitespace, not at
1427# punctuation that may be part of a password -- except that ONE trailing
1428# ``,`` or ``;`` directly before whitespace or the end of the text is left
1429# as the list delimiter it almost always is (``max token = 4096, temp``).
1430# An unquoted value also ends where another credential assignment starts,
1431# but only where that assignment, redacted in its own right, covers
1432# everything the longer value would have covered and leaves nothing after
1433# it that a later pass would have redacted. Stripping control characters
1434# glues such an assignment onto the value before it
1435# (``token = [REDACTED]\nap<U+200B>i_key = x`` becomes
1436# ``token = [REDACTED]api_key = x``); without the stop, the value of
1437# ``token`` would run on and take the name ``api_key``, leaving ``x``
1438# visible. The next assignment qualifies in these shapes only:
1439#
1440# * ``name=`` with no whitespace in its match: an unquoted value, or a
1441# quoted value without whitespace that closes or runs to the end of the
1442# text. Its match ends at the same whitespace the longer value would
1443# have, which must be one ``strip_control_chars`` keeps (or the end of
1444# the text): at a stripped line break the patcher's second pass would
1445# glue the next line onto the visible name.
1446# * a quoted value that never closes: its match takes the rest of the text.
1447# * whitespace around the ``=`` or inside a quoted value
1448# (``token=x;password="p q" r``): its match then runs on past the
1449# whitespace that ended the longer value, over text the scan after it
1450# would otherwise read. So that text must hold nothing such a scan, or a
1451# later pass, could redact differently: no character
1452# ``strip_control_chars`` removes (a later pass would glue it to its
1453# neighbours), no ``name=`` inside a quoted value (the value would close on
1454# that name's opening quote, leaving the rest of its value visible), and
1455# in an unquoted value, or in the text glued after a closing quote up to
1456# the next whitespace, no ``name=`` and no ``Bearer``, ``Authorization:``
1457# or ``x-api-key:`` (anchors whose value lies past that whitespace), which
1458# must again be kept whitespace or the end of the text.
1459#
1460# Anything else is not a stop, and the longer value runs on as before.
1461_DOUBLE_QUOTED_VALUE = r"\"(?:\\.|[^\"\\])*"
1462_SINGLE_QUOTED_VALUE = r"'(?:\\.|[^'\\])*"
1463# A character ``strip_control_chars`` keeps, and kept whitespace.
1464_KEPT = r"(?!{})".format(_UNSAFE_CHAR_RE.pattern)
1465_KEPT_SPACE = _KEPT + r"\s"
1466_ENDS_AT_KEPT_SPACE = r"(?={}|\Z)".format(_KEPT_SPACE)
1467# How far past a candidate stop its lookahead reads to the end of a run of
1468# non-whitespace. Each candidate in one run reads to the same end, so the
1469# bound keeps a run dense with credential names linear; a value that runs
1470# on longer is not a stop.
1471_STOP_LOOKAHEAD_CHARS = 256
1472_GAP = r"(?:\s|{})*".format(_UNSAFE_CHAR_RE.pattern)
1473# An assignment as a pass reads it, before or after that strip: a credential
1474# name where an assignment can start, then ``=``, with any whitespace or
1475# character ``strip_control_chars`` removes between them.
1476_NAME_THEN_EQUALS = (
1477 r"(?<![\w.?&-])(?:" + _STOP_SENSITIVE_NAME_PATTERN + r")" + _GAP + r"="
1478)
1479# The same, or the anchor of a credential shape whose value may lie past
1480# whitespace (``Authorization: Basic ...``, ``Bearer ...``, ``x-api-key: ...``).
1481_NAME_OR_SHAPE_ANCHOR = (
1482 _NAME_THEN_EQUALS + r"|bearer|(?:authorization|x-api-key)" + _GAP + r"[:=]"
1483)
1486# A run of non-whitespace up to kept whitespace or the end of the text,
1487# holding no assignment or credential-shape anchor.
1488_PLAIN_RUN = (
1489 r"(?:(?!"
1490 + _NAME_OR_SHAPE_ANCHOR
1491 + r")"
1492 + _KEPT
1493 + r"\S){0,"
1494 + str(_STOP_LOOKAHEAD_CHARS)
1495 + r"}"
1496 + _ENDS_AT_KEPT_SPACE
1497)
1500def _spread_quoted_value(quote: str) -> str:
1501 """A quoted value whose match may reach past whitespace (see above)."""
1502 return (
1503 quote
1504 + r"(?:(?!"
1505 + _NAME_THEN_EQUALS
1506 + r")"
1507 + _KEPT
1508 + r"(?:\\"
1509 + _KEPT
1510 + r"\S|[^"
1511 + quote
1512 + r"\\]))*(?:"
1513 + quote
1514 + r"(?="
1515 + _PLAIN_RUN
1516 + r")|\\?\Z)"
1517 )
1520def _space_before_closing(quote: str) -> str:
1521 """Lookahead: whitespace inside the quoted value that starts here."""
1522 return r"(?=" + quote + r"(?:\\\S|[^" + quote + r"\\\s])*\s)"
1525# An unquoted value that may reach past whitespace (see above).
1526_SPREAD_UNQUOTED_VALUE = r"(?=[^\s\"'])" + _PLAIN_RUN
1527_NEXT_ASSIGNMENT = (
1528 r"(?<![\w.?&-])(?:" + _STOP_SENSITIVE_NAME_PATTERN + r")"
1529 # ``name=`` with no whitespace in its match.
1530 r"(?:=(?:\"(?:\\[^\s]|[^\"\\\s])*(?:\"|\\?\Z)"
1531 r"|'(?:\\[^\s]|[^'\\\s])*(?:'|\\?\Z)"
1532 r"|[^\s\"']\S{0,"
1533 + str(_STOP_LOOKAHEAD_CHARS)
1534 + r"})"
1535 + _ENDS_AT_KEPT_SPACE
1536 # A quoted value that never closes: its match takes the rest of the text.
1537 + r"|\s*=\s*(?:"
1538 + _DOUBLE_QUOTED_VALUE
1539 + r"|"
1540 + _SINGLE_QUOTED_VALUE
1541 + r")\\?$"
1542 # Whitespace around the ``=``.
1543 + r"|(?:(?:"
1544 + _KEPT_SPACE
1545 + r")+=(?:"
1546 + _KEPT_SPACE
1547 + r")*|=(?:"
1548 + _KEPT_SPACE
1549 + r")+)(?:"
1550 + _SPREAD_UNQUOTED_VALUE
1551 + r"|"
1552 + _spread_quoted_value('"')
1553 + r"|"
1554 + _spread_quoted_value("'")
1555 + r")"
1556 # Whitespace inside a quoted value.
1557 + r"|="
1558 + _space_before_closing('"')
1559 + _spread_quoted_value('"')
1560 + r"|="
1561 + _space_before_closing("'")
1562 + _spread_quoted_value("'")
1563 + r")"
1564)
1565_ASSIGNMENT_SEPARATOR_PATTERN = r"(\s*=\s*)"
1566_ASSIGNMENT_VALUE_PATTERN = (
1567 r"(?:\"(?:\\.|[^\"\\])*(?:\"|\\?$)|'(?:\\.|[^'\\])*(?:'|\\?$)"
1568 r"|\S+?(?=[,;]?(?:\s|$)))"
1569)
1570_STOP_ASSIGNMENT_VALUE_PATTERN = (
1571 r"(?:\"(?:\\.|[^\"\\])*(?:\"|\\?$)|'(?:\\.|[^'\\])*(?:'|\\?$)"
1572 r"|(?:(?!"
1573 + _NEXT_ASSIGNMENT
1574 + r")\S)+?(?=(?:"
1575 + _NEXT_ASSIGNMENT
1576 + r"|[,;]?(?:\s|$))))"
1577)
1578_ASSIGNMENT_REPLACEMENT = r"\1\2" + REDACTION_TEXT
1579_ASSIGNMENT_REPLACEMENT_PARTS = (1, 2, REDACTION_TEXT)
1581# The exact names (``password``, ``client_secret``, ``api-key``). Every
1582# pass of the log redaction runs this before the prefixed-name pattern
1583# below, which skips these names. Run after it, the prefixed pattern only
1584# replaces more of the text: a prefixed value that swallows the start of an
1585# exact-name assignment (``OPENAI_API_KEY=abc,password = x``) no longer
1586# hides it. Across the two passes of :func:`_redact_log_text`'s
1587# "restricted first pass" pipeline the same holds through
1588# :data:`_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE`.
1589_BARE_SENSITIVE_ASSIGNMENT_RE = re.compile(
1590 r"(?i)(?<![\w.?&-])("
1591 + _LOG_SENSITIVE_NAME_PATTERN
1592 + r")"
1593 + _ASSIGNMENT_SEPARATOR_PATTERN
1594 + _ASSIGNMENT_VALUE_PATTERN
1595)
1596_STOP_BARE_SENSITIVE_ASSIGNMENT_RE = re.compile(
1597 r"(?i)(?<![\w.?&-])("
1598 + _LOG_SENSITIVE_NAME_PATTERN
1599 + r")"
1600 + _ASSIGNMENT_SEPARATOR_PATTERN
1601 + _STOP_ASSIGNMENT_VALUE_PATTERN
1602)
1604# Names the prefixed pattern must never match, checked by a lookahead
1605# BEFORE anything is consumed, so the regex scans on inside their values:
1606# - an exact name: the bare pass has already replaced its value, and
1607# matching it again would let the unquoted rule run that value on into a
1608# glued name (``client_secret='x'llm.api_key='y z'``) and hide it;
1609# - the one leaf settings redaction exempts (``requires_api_key``, a
1610# boolean flag; see ``data_sanitizer._has_non_secret_qualifier``),
1611# following a ``.`` or starting the name, as that leaf is computed there.
1612# A match that consumed the flag's value and handed it back unchanged
1613# would make ``re.sub`` resume after that value, so an assignment inside
1614# it (``requires_api_key=true,password=x``, or an unterminated quote
1615# running to the end of the text) would never be scanned.
1616_EXEMPT_ASSIGNMENT_NAME_PATTERN = "".join(
1617 (
1618 r"(?:(?:(?:[a-z0-9]+[._])*[a-z0-9]+\.)?",
1619 re.escape(_EXEMPT_QUALIFIER_LEAF).replace("_", "[_-]"),
1620 r")",
1621 )
1622)
1623_NOT_PREFIXED_ASSIGNMENT_NAME_PATTERN = (
1624 r"(?!(?:"
1625 + _EXEMPT_ASSIGNMENT_NAME_PATTERN
1626 + r"|(?:"
1627 + _LOG_SENSITIVE_NAME_PATTERN
1628 + r"))\s*=)"
1629)
1630_PREFIXED_ASSIGNMENT_NAME_PATTERN = (
1631 # ``(?=[a-z0-9])`` first: rejects a non-name position before the
1632 # lookahead's long alternation runs (a run of spaces stays cheap).
1633 r"(?i)(?<![\w.?&-])(?=[a-z0-9])"
1634 + _NOT_PREFIXED_ASSIGNMENT_NAME_PATTERN
1635 + r"((?:[a-z0-9]+[._])*(?:"
1636 + _LOG_SENSITIVE_NAME_PATTERN
1637 + r"))"
1638 + _ASSIGNMENT_SEPARATOR_PATTERN
1639)
1640_SENSITIVE_ASSIGNMENT_RE = re.compile(
1641 _PREFIXED_ASSIGNMENT_NAME_PATTERN + _ASSIGNMENT_VALUE_PATTERN
1642)
1643_STOP_SENSITIVE_ASSIGNMENT_RE = re.compile(
1644 _PREFIXED_ASSIGNMENT_NAME_PATTERN + _STOP_ASSIGNMENT_VALUE_PATTERN
1645)
1647# Exact and prefixed names in ONE pattern, so whichever assignment starts
1648# first wins. Two readings of it run, each as a pipeline of
1649# :func:`_redact_log_text`:
1650# - "next-line names" (:data:`_LEFTMOST_SENSITIVE_ASSIGNMENT_RE`, first
1651# pass only, on the original separators; also a pipeline of
1652# :func:`redact_log_message`): the exempt flag is skipped by
1653# the lookahead, and a name at the start of the next line followed by
1654# ``=`` is never taken as a value: ``DB_PASSWORD=\nOPENAI_API_KEY = x``
1655# is an empty ``DB_PASSWORD`` and an ``OPENAI_API_KEY`` whose value is
1656# ``x``. Read otherwise, the name is the value and `` = x`` is left
1657# unredacted. "The next line" is any run of whitespace and
1658# :func:`strip_control_chars` characters between the ``=`` and the name
1659# that holds at least one of the latter (``\n``, ``\r\n``, ``\x1c``,
1660# U+2028, ...): the indentation of a pretty-printed dump
1661# (``DB_PASSWORD =\n OPENAI_API_KEY = x``) and an invisible character
1662# before the name (``\n<U+200B>OPENAI_API_KEY``) included. A name after
1663# only spaces on the same line (``password= api_key = x``) is still read
1664# as the value.
1665# - "flag kept whole" (:data:`_START_ORDER_SENSITIVE_ASSIGNMENT_RE`, both
1666# passes): the exempt flag's assignment is matched like any other and
1667# kept as it is, value included, so nothing in its value starts an
1668# assignment (``requires_api_key=a,db_password\n=x;token=y``: the value
1669# ``a,db_password`` is the flag's, and ``token=y`` is still found).
1670_ANY_SENSITIVE_NAME_PATTERN = (
1671 r"(?=[a-z0-9])(?!"
1672 + _EXEMPT_ASSIGNMENT_NAME_PATTERN
1673 + r"\s*=)(?:[a-z0-9]+[._])*(?:"
1674 + _LOG_SENSITIVE_NAME_PATTERN
1675 + r")"
1676)
1677# Whitespace that :func:`strip_control_chars` keeps: a space, U+00A0,
1678# U+3000, ... -- not a tab or a line break, which it removes.
1679_KEPT_WHITESPACE_PATTERN = "".join(
1680 (r"(?:(?!", _UNSAFE_CHAR_RE.pattern, r")\s)")
1681)
1682# The next-line guard of :data:`_LEFTMOST_SENSITIVE_ASSIGNMENT_RE`, checked
1683# right after the ``=``: kept whitespace, one stripped character, then any
1684# mix of whitespace and stripped characters (ONE character class --
1685# ``_UNSAFE_CHAR_RE`` is a single class, whose inside is reused -- so a
1686# long run is scanned once, not re-split), then a name and ``=``. The parts
1687# before the name are disjoint or a single class, so the check is linear in
1688# the run it scans.
1689_NEXT_LINE_NAME_GUARD_PATTERN = (
1690 r"(?!"
1691 + _KEPT_WHITESPACE_PATTERN
1692 + r"*"
1693 + _UNSAFE_CHAR_RE.pattern
1694 + r"[\s"
1695 + _UNSAFE_CHAR_RE.pattern[1:-1]
1696 + r"]*"
1697 + _ANY_SENSITIVE_NAME_PATTERN
1698 + r"\s*=)"
1699)
1700_LEFTMOST_SENSITIVE_ASSIGNMENT_RE = re.compile(
1701 r"(?i)(?<![\w.?&-])("
1702 + _ANY_SENSITIVE_NAME_PATTERN
1703 + r")(\s*="
1704 + _NEXT_LINE_NAME_GUARD_PATTERN
1705 + r"\s*)"
1706 + _ASSIGNMENT_VALUE_PATTERN
1707)
1708_START_ORDER_SENSITIVE_ASSIGNMENT_RE = re.compile(
1709 r"(?i)(?<![\w.?&-])((?:[a-z0-9]+[._])*(?:"
1710 + _LOG_SENSITIVE_NAME_PATTERN
1711 + r"))"
1712 + _ASSIGNMENT_SEPARATOR_PATTERN
1713 + _ASSIGNMENT_VALUE_PATTERN
1714)
1715# Run the guarded stop as one additional reading, leaving the established
1716# pipelines intact. Their union keeps every value the old readings masked
1717# while the stop reading can preserve and redact a glued next name.
1718_STOP_START_ORDER_SENSITIVE_ASSIGNMENT_RE = re.compile(
1719 r"(?i)(?<![\w.?&-])((?:[a-z0-9]+[._])*(?:"
1720 + _LOG_SENSITIVE_NAME_PATTERN
1721 + r"))"
1722 + _ASSIGNMENT_SEPARATOR_PATTERN
1723 + _STOP_ASSIGNMENT_VALUE_PATTERN
1724)
1725_EXEMPT_ASSIGNMENT_NAME_RE = re.compile(
1726 r"(?i)" + _EXEMPT_ASSIGNMENT_NAME_PATTERN + r"\Z"
1727)
1730def _start_order_replacement_parts(match: "re.Match[str]") -> tuple:
1731 """Replacement for :data:`_START_ORDER_SENSITIVE_ASSIGNMENT_RE`, as parts.
1733 The exempt flag's whole assignment is kept (group 0); any other keeps
1734 its name and separator and redacts the value.
1735 """
1736 if _EXEMPT_ASSIGNMENT_NAME_RE.match(match.group(1)):
1737 return (0,)
1738 return _ASSIGNMENT_REPLACEMENT_PARTS
1741def _redact_start_order_match(match: "re.Match[str]") -> str:
1742 """Replacement for :data:`_START_ORDER_SENSITIVE_ASSIGNMENT_RE`."""
1743 return _join_replacement_parts(match, _start_order_replacement_parts)
1746# The prefixed pattern for the first pass of the "restricted" pipeline of
1747# :func:`_redact_log_text`. It must not change how that pipeline's second
1748# pass parses the text around the value it replaces, or that pass -- whose
1749# bare-name half is exactly what the bare names alone would redact -- could
1750# match less. So it only takes a value the second pass could never read
1751# differently:
1752# - no ``=`` and not followed by ``\s*=``, so no name inside it (one that
1753# only becomes recognisable once invisible characters are stripped,
1754# ``pass<U+200B>word = x``) is lost;
1755# - followed by whitespace that survives :func:`strip_control_chars` (or by
1756# the end of the text), so the stripped text never glues the replacement
1757# onto the next token and makes a name there newly recognisable, whose
1758# value could then swallow the next one;
1759# - unquoted with no quote character in it, or quoted with no quote
1760# character or backslash inside the quotes -- and a quoted value keeps
1761# its quotes (``"[REDACTED]"``) -- so a quote the second pass opens
1762# earlier still closes where it would have. The second pass then
1763# redacts the kept quotes along with the value.
1764# Any other value is left for the second pass, which can MISS it: on the
1765# stripped text a name the newline separated is glued to the line before
1766# (``-\nDB_PASSWORD=x`` reads ``-DB_PASSWORD=x``, a name after ``-``), or
1767# the value before it runs on into it. That is why this pipeline never
1768# runs alone: the others read the first kind on the original separators,
1769# where the newline still separates the name. The second kind is missed by
1770# every reading -- each reads the quoted value's start as part of the value
1771# before it, and none redacts the rest -- when this pattern cannot take it
1772# either: a prefixed name's quoted value followed by a line break, a tab
1773# or another stripped character (``OPENAI_API_KEY=abc,DB_PASSWORD="a b c"``
1774# and ``\n`` leaves `` b c"``), by any other non-whitespace character
1775# (``")``) or by whitespace and ``=`` (``" = x``); one whose run-on value
1776# belongs to an exact name, which this pipeline's bare-name step has
1777# already redacted to the end of the quote's first word
1778# (``token=abc,DB_PASSWORD="a b c"``); or a quoted assignment glued on
1779# with ``&``, ``?`` or ``-`` (``OPENAI_API_KEY=abc&DB_PASSWORD="a b c"``,
1780# also with an exact name), which no reading takes as a quoted value.
1781_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE = re.compile(
1782 _PREFIXED_ASSIGNMENT_NAME_PATTERN
1783 + r"(?:([\"'])[^\"'\\=]*(\3|\Z)|[^\s=\"']+?)"
1784 + r"(?=[,;]?(?:"
1785 + _KEPT_WHITESPACE_PATTERN
1786 + r"|\Z))(?!\s*=)"
1787)
1790def _first_pass_replacement_parts(match: "re.Match[str]") -> tuple:
1791 """Replacement for :data:`_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE`, as parts.
1793 The name, the separator and any quotes are kept (group numbers); only
1794 the value becomes :data:`REDACTION_TEXT` (a literal). See
1795 :func:`_substitute_tracked` for the part format.
1796 """
1797 parts: list = [1, 2]
1798 if match.group(3):
1799 parts.append(3)
1800 parts.append(REDACTION_TEXT)
1801 if match.group(4):
1802 parts.append(4)
1803 return tuple(parts)
1806def _join_replacement_parts(match: "re.Match[str]", parts_for) -> str:
1807 """The replacement text *parts_for* gives for *match*."""
1808 return "".join(
1809 part if isinstance(part, str) else match.group(part)
1810 for part in parts_for(match)
1811 )
1814def _redact_first_pass_match(match: "re.Match[str]") -> str:
1815 """Replacement for :data:`_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE`."""
1816 return _join_replacement_parts(match, _first_pass_replacement_parts)
1819def redact_sensitive_assignments(message: str) -> str:
1820 """Redact known credential assignments with quoted or unquoted values.
1822 Complements :func:`sanitize_error_message` (which only redacts a
1823 ``field=value`` when it sits in a URL query string, i.e. preceded by
1824 ``?``/``&``). The field name (``api_key=``, ``password=``, ``token=``,
1825 ``OPENAI_API_KEY=``, ``llm.openai.api_key=``, ...) is preserved; only
1826 its value is replaced. Returns *message*
1827 unchanged when it is empty or contains no sensitive assignment.
1828 """
1829 message = _plain_text(message)
1830 return _scrub_to_fixed_point(_redact_sensitive_assignments_once, message)
1833def _redact_sensitive_assignments_once(message: str) -> str:
1834 if not message: 1834 ↛ 1835line 1834 didn't jump to line 1835 because the condition on line 1834 was never true
1835 return message
1836 bare = _STOP_BARE_SENSITIVE_ASSIGNMENT_RE.sub(
1837 _ASSIGNMENT_REPLACEMENT, message
1838 )
1839 return _STOP_SENSITIVE_ASSIGNMENT_RE.sub(_ASSIGNMENT_REPLACEMENT, bare)
1842# Upper bound on how much of ONE string the log-path redaction scans. The
1843# patcher runs synchronously in the logging thread on every record (message,
1844# every ``extra`` string leaf, every rendered exception field), so its cost
1845# must not grow with whatever a call site hands it -- fetched page text, a
1846# response body, a long traceback string. Text past the bound is NOT passed
1847# through unscanned (that would ship any secret in it); it is dropped and
1848# replaced by a marker giving the omitted length, in every sink (stderr,
1849# file, DB, frontend). 32 KiB keeps a fetched page (the fetch tool logs up
1850# to 10,000 characters) intact. On adversarial 32 KiB input the passes of
1851# :func:`_redact_log_text` measured up to ~280 ms together on an idle
1852# machine and about twice that on a loaded one, and those of
1853# :func:`redact_log_message` up to ~240 ms idle (``pwd="a"b=c `` repeated):
1854# a text whose pipelines disagree on every line (that one,
1855# ``db_password = a, `` repeated, or ``DB_PASSWORD=x`` lines) runs all of
1856# them with origins tracked, several times the cost of one pipeline. A
1857# text with no credential name followed by ``=`` skips the name steps (see
1858# :func:`_result_without_name_assignments`): about 10-50 ms at 32 KiB.
1859# The cost is linear in the length scanned (8x the time for 8x the text,
1860# measured from 4 KiB to 32 KiB), so it is the same for a 1 MB or a 10 MB
1861# string; the cut itself is one scan of at most 32 KiB (under 0.3 ms).
1862_LOG_REDACTION_MAX_CHARS = 32_768
1863# The run at the end of a truncated head, matched reversed: non-whitespace
1864# plus every character ``strip_control_chars`` removes (``\n``, ``\t``,
1865# ``\x1c``, U+0085, U+2028, ...). ``_redact_log_text`` strips those between
1866# its passes, which rejoins a token they split, so they are not a boundary.
1867# Built from ``_UNSAFE_CHAR_RE`` itself so the two cannot drift apart.
1868_LOG_TRAILING_RUN_RE = re.compile(r"(?:\S|" + _UNSAFE_CHAR_RE.pattern + r")*")
1869# Digits bounded (ASCII, at most 19) so a forged marker cannot make
1870# ``int()`` exceed its digit limit; the real count is far below that.
1871_LOG_OMISSION_MARKER_RE = re.compile(
1872 r" \[\.\.\. ([0-9]{1,19}) characters omitted from log output\]\Z"
1873)
1876def _truncate_for_log_redaction(
1877 message: str, max_chars: int = _LOG_REDACTION_MAX_CHARS
1878) -> tuple[str, str]:
1879 """Split *message* into the part to scan and a marker for the rest.
1881 *max_chars* is the bound (the patcher's ``_LOG_REDACTION_MAX_CHARS``
1882 unless a caller that shows only a short prefix of the result passes a
1883 smaller one, see :func:`redact_and_bound_for_log`); the rule below is
1884 the same for any bound.
1886 The cut only lands at a boundary: a whitespace character that
1887 :func:`strip_control_chars` keeps (a space, U+00A0, U+3000, ...). A
1888 control or separator whitespace character (``\n``, ``\t``, ``\x1c``,
1889 U+0085, U+2028, ...) is not one, because :func:`_redact_log_text`
1890 strips it between its passes and so joins the text on either side.
1891 Unless the character at the bound is a boundary, the cut moves back to
1892 just after the last boundary before the bound, dropping the whole run
1893 the bound falls in -- however long, so a string with no boundary in its
1894 first *max_chars* characters keeps nothing but the
1895 marker. A partial run could be a credential fragment its shape pattern
1896 no longer recognises (a key cut below its minimum length, URL userinfo
1897 cut before its ``@``, a password cut at a ``&``). A credential token
1898 contains no boundary character -- one split by a stripped control
1899 character is a single run here, as it is to the second pass -- so the
1900 head keeps at most a label that the passes redact or that holds no
1901 secret (``password = ``, ``Authorization: Basic ``, or an unterminated
1902 quote, which the assignment pass redacts to the end of the head, so a
1903 quoted value with spaces in it is covered too).
1904 """
1905 if len(message) <= max_chars:
1906 return message, ""
1907 cut = max_chars
1908 at_bound = message[cut]
1909 if not at_bound.isspace() or _UNSAFE_CHAR_RE.match(at_bound):
1910 # ``\S`` is exactly ``not str.isspace()``; matching the reversed head
1911 # measures the trailing run in one linear scan.
1912 run = _LOG_TRAILING_RUN_RE.match(message[cut - 1 :: -1])
1913 cut -= run.end()
1914 omitted = len(message) - cut
1915 return message[:cut], f" [... {omitted} characters omitted from log output]"
1918def log_length_before_truncation(message: str) -> int:
1919 """Length *message* had before the log-path length bound cut it.
1921 For a string ending in the marker :func:`redact_log_message` appends,
1922 the kept text plus the omitted count; otherwise ``len(message)``. Both
1923 are measured after redaction, which may change the length. Never
1924 raises: a marker the pattern does not accept (logged text can forge
1925 one) yields ``len(message)``.
1926 """
1927 marker = _LOG_OMISSION_MARKER_RE.search(message)
1928 if marker is None:
1929 return len(message)
1930 try:
1931 return marker.start() + int(marker.group(1))
1932 except ValueError:
1933 return len(message)
1936def redact_log_message(message: str) -> str:
1937 """Full credential scrub for a log *message* string.
1939 Runs the credential-*shape* patterns :func:`sanitize_error_message`
1940 runs (its ``_CREDENTIAL_PATTERNS``: URL userinfo, ``?param=``
1941 credentials, ``Bearer``/``x-api-key`` headers, well-known key prefixes,
1942 JWTs) -- the shape redaction the sanctioned :func:`scrub_error` applies,
1943 run as a backstop on every record -- then ``?name=``/``&name=`` query
1944 parameters for every :data:`LOG_SENSITIVE_KEYS` name, then
1945 ``name=value`` assignments of those names, exact or prefixed (the
1946 names :func:`redact_sensitive_assignments` recognises), in each reading
1947 of :data:`_LOG_MESSAGE_PIPELINES`; what any reading redacts is
1948 redacted. Control characters are kept (exception text is rendered
1949 with its newlines). Non-sensitive text up to
1950 ``_LOG_REDACTION_MAX_CHARS`` characters is returned unchanged; a longer
1951 string is cut there (see :func:`_truncate_for_log_redaction`) and the
1952 remainder replaced by an omitted-length marker, so the cost per string
1953 is bounded.
1954 """
1955 message = _plain_text(message)
1956 if not message:
1957 return _exact_str(message)
1958 head, marker = _truncate_for_log_redaction(message)
1959 return _scrub_to_fixed_point(_redact_bounded_log_text, head) + marker
1962# One substitution of a redaction pass: a pattern and its replacement,
1963# given as the string or callable ``re.sub`` takes and as PARTS -- a list
1964# (or a callable giving one per match) of group numbers, which keep that
1965# group's text, and literal strings, which are inserted. The parts let
1966# :func:`_substitute_tracked` follow every kept character back to the
1967# position it came from.
1968_TEMPLATE_GROUP_RE = re.compile(r"\\([0-9])")
1971def _template_parts(template: str) -> tuple:
1972 """Parts of a ``re.sub`` template that uses only ``\\1``-style groups."""
1973 if "\\" in _TEMPLATE_GROUP_RE.sub("", template): 1973 ↛ 1974line 1973 didn't jump to line 1974 because the condition on line 1973 was never true
1974 raise ValueError(f"unsupported replacement template: {template!r}")
1975 parts: list = []
1976 for index, piece in enumerate(_TEMPLATE_GROUP_RE.split(template)):
1977 if index % 2:
1978 parts.append(int(piece))
1979 elif piece:
1980 parts.append(piece)
1981 return tuple(parts)
1984def _step(pattern: "re.Pattern[str]", replacement: Any, parts: Any = None):
1985 if parts is None:
1986 parts = _template_parts(replacement)
1987 return pattern, replacement, parts
1990# The credential-shape and query-parameter steps every pass starts with.
1991_SHAPE_STEPS = tuple(
1992 _step(pattern, replacement) for pattern, replacement in _CREDENTIAL_PATTERNS
1993) + (_step(_SENSITIVE_QUERY_RE, r"\1" + REDACTION_TEXT),)
1994_BARE_STEP = _step(_BARE_SENSITIVE_ASSIGNMENT_RE, _ASSIGNMENT_REPLACEMENT)
1995_PREFIXED_STEP = _step(_SENSITIVE_ASSIGNMENT_RE, _ASSIGNMENT_REPLACEMENT)
1996# The name steps of :func:`redact_log_message`'s pass before the glued
1997# tails were closed (below). The bare names run before the prefixed ones
1998# (see :data:`_BARE_SENSITIVE_ASSIGNMENT_RE`); the prefixed ones run twice,
1999# because a quoted value glued to the next text (``OPENAI_API_KEY="x"tail``)
2000# leaves ``[REDACTED]tail``, which a second scan reads as one unquoted value.
2001# An exact name's tail (``password="x"tail``) was left.
2002_NAME_STEPS_BEFORE_CLOSING = (_BARE_STEP, _PREFIXED_STEP, _PREFIXED_STEP)
2003# A quoted value glued to the next text leaves that text behind when it is
2004# redacted: ``password="x"tail`` becomes ``password=[REDACTED]tail``. This
2005# step reads such an assignment again, as a scan for names would (the same
2006# names, word start and separator; not the exempt flag), but only where the
2007# value starts with ``[REDACTED]``, and redacts the tail up to the
2008# whitespace (or ``,``/``;`` before it) where an unquoted value ends --
2009# unless the tail holds an ``=`` or is followed by one, i.e. is itself the
2010# name of an assignment (``passwd=""apikey =""tail``: ``apikey`` stays,
2011# its own ``tail`` goes). So, unlike another scan for names, it never reads
2012# a value on into the name of an assignment whose own tail it has not yet
2013# redacted.
2014_GLUED_TAIL_RE = re.compile(
2015 r"(?i)(?<![\w.?&-])("
2016 + _ANY_SENSITIVE_NAME_PATTERN
2017 + r")"
2018 + _ASSIGNMENT_SEPARATOR_PATTERN
2019 + re.escape(REDACTION_TEXT)
2020 + r"(?![,;]?(?:\s|$))[^\s=]+?(?=[,;]?(?:\s|$))(?!\s*=)"
2021)
2022_GLUED_TAIL_STEP = _step(_GLUED_TAIL_RE, _ASSIGNMENT_REPLACEMENT)
2023# The "exact names first" name steps: the bare names, the prefixed ones,
2024# the glued tails, then both name scans once more, the bare names last
2025# (see :data:`_BARE_SENSITIVE_ASSIGNMENT_RE`). The last two scans redact a
2026# tail the tail step leaves (one holding an ``=``: ``password="x"a=b``) as
2027# part of an unquoted value. Running these steps again on their result
2028# changes nothing (seeded fuzz: 0 of 50,000 messages).
2029_FULL_PASS_NAME_STEPS = (
2030 _BARE_STEP,
2031 _PREFIXED_STEP,
2032 _GLUED_TAIL_STEP,
2033 _PREFIXED_STEP,
2034 _BARE_STEP,
2035)
2036_FULL_PASS_STEPS = _SHAPE_STEPS + _FULL_PASS_NAME_STEPS
2037_RESTRICTED_FIRST_PASS_STEPS = (
2038 _BARE_STEP,
2039 _step(
2040 _FIRST_PASS_SENSITIVE_ASSIGNMENT_RE,
2041 _redact_first_pass_match,
2042 _first_pass_replacement_parts,
2043 ),
2044)
2045_LEFTMOST_STEP = _step(
2046 _LEFTMOST_SENSITIVE_ASSIGNMENT_RE, _ASSIGNMENT_REPLACEMENT
2047)
2048# The "next-line names" steps, closed the same way, so
2049# ``DB_PASSWORD=\nOPENAI_API_KEY = "x"tail`` does not leave ``tail`` -- which
2050# the second pass, reading ``OPENAI_API_KEY`` as ``DB_PASSWORD``'s value on
2051# the stripped text, would not redact either.
2052_LEFTMOST_STEPS = (_LEFTMOST_STEP, _GLUED_TAIL_STEP, _LEFTMOST_STEP)
2053_START_ORDER_STEPS = (
2054 _step(
2055 _START_ORDER_SENSITIVE_ASSIGNMENT_RE,
2056 _redact_start_order_match,
2057 _start_order_replacement_parts,
2058 ),
2059)
2060_STOP_START_ORDER_STEPS = (
2061 _step(
2062 _STOP_START_ORDER_SENSITIVE_ASSIGNMENT_RE,
2063 _redact_start_order_match,
2064 _start_order_replacement_parts,
2065 ),
2066)
2067# The pipelines of :func:`_redact_log_text`: the name steps each runs after
2068# :data:`_SHAPE_STEPS`, in its first pass (on the original text) and
2069# in its second (on the text with control characters stripped). The first
2070# pipeline's text is the one kept; the others only add redactions to it.
2071_LOG_TEXT_PIPELINES = (
2072 # "exact names first": :func:`redact_log_message`'s name steps, on the
2073 # original separators and again on the stripped text. Its result is
2074 # the one kept.
2075 (_FULL_PASS_NAME_STEPS, _FULL_PASS_NAME_STEPS),
2076 # "restricted first pass": a first pass that never changes how the
2077 # second reads the text around a value it replaces (see
2078 # :data:`_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE`).
2079 (_RESTRICTED_FIRST_PASS_STEPS, _FULL_PASS_NAME_STEPS),
2080 # "next-line names": names in start order, in the first pass (see
2081 # :data:`_LEFTMOST_SENSITIVE_ASSIGNMENT_RE`).
2082 (_LEFTMOST_STEPS, _FULL_PASS_NAME_STEPS),
2083 # "flag kept whole": names in start order, in both passes (see
2084 # :data:`_START_ORDER_SENSITIVE_ASSIGNMENT_RE`).
2085 (_START_ORDER_STEPS, _START_ORDER_STEPS),
2086 # "exact names only": the redaction as it was before the prefix rule
2087 # (with the trailing ``,``/``;`` rule of the value pattern), so this
2088 # one never removes less than the exact names alone would.
2089 ((_BARE_STEP,), (_BARE_STEP,)),
2090 # "exact names first" and "next-line names" with their first pass as
2091 # it was before the glued tails were closed. Closing them changes the
2092 # text the later steps of that pass read, and removes text before the
2093 # strip, which can join what follows onto a different neighbour and
2094 # change how the second pass reads it; with these readings kept,
2095 # closing never shows anything that leaving it open would hide.
2096 (_NAME_STEPS_BEFORE_CLOSING, _FULL_PASS_NAME_STEPS),
2097 ((_LEFTMOST_STEP,), _FULL_PASS_NAME_STEPS),
2098 # Preserve names that an earlier value would otherwise swallow.
2099 (_STOP_START_ORDER_STEPS, _STOP_START_ORDER_STEPS),
2100)
2101# The pipelines of :func:`redact_log_message`, which also renders exception
2102# text (``str()``, notes, ``SyntaxError`` fields) and so keeps every
2103# control character: the first passes of :data:`_LOG_TEXT_PIPELINES`, in
2104# that order, on the text as it is -- one pass each, no second pass
2105# (``None``) -- except that the "restricted first pass" reading, which
2106# leaves values for its second pass, runs the "exact names first" steps
2107# after it on the same text, and the unclosed "next-line names" reading is
2108# left out: its closed steps start with it and only run more steps after
2109# it on the same text, so they only redact more. So an empty
2110# ``DB_PASSWORD=`` at the end of a line does not take the name starting the
2111# next line as its value, and ``OPENAI_API_KEY=abc,DB_PASSWORD="a b c"``
2112# followed by the end of the text or by a space (with no ``=`` after it)
2113# does not leave `` b c"`` readable, in an exception's text as in a log
2114# message (followed by a line break, or by `` = x``, it does in both; see
2115# :data:`_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE`).
2116_LOG_MESSAGE_PIPELINES = (
2117 (_FULL_PASS_NAME_STEPS, None),
2118 (_NAME_STEPS_BEFORE_CLOSING, None),
2119 (_RESTRICTED_FIRST_PASS_STEPS + _FULL_PASS_NAME_STEPS, None),
2120 (_LEFTMOST_STEPS, None),
2121 (_START_ORDER_STEPS, None),
2122 ((_BARE_STEP,), None),
2123 (_STOP_START_ORDER_STEPS, None),
2124)
2127class _BarePass(NamedTuple):
2128 """A name-based log pass and its unanchored variant for glued text."""
2130 pattern: "re.Pattern[str]"
2131 replacement: str
2132 name_group: int
2133 value_after_group: int
2134 unanchored: "re.Pattern[str]"
2137def _without_name_boundary(pattern: "re.Pattern[str]") -> "re.Pattern[str]":
2138 return re.compile(pattern.pattern.replace(r"(?<![\w.?&-])", "", 1))
2141def _log_patcher_bare_passes() -> tuple[_BarePass, ...]:
2142 """Find the query and name patterns in every live log pipeline.
2144 The control-splice guard must follow new name passes automatically;
2145 otherwise a splice could consume a name and expose its value to a sink.
2146 All current name passes capture the name and separator as groups 1/2.
2147 The replacement here is used only to detect whether a prospective match
2148 would hide anything, so a conservative value marker is sufficient for
2149 passes with conditional replacement callbacks.
2150 """
2151 passes = [
2152 _BarePass(
2153 _SENSITIVE_QUERY_RE,
2154 r"\1" + REDACTION_TEXT,
2155 1,
2156 1,
2157 _SENSITIVE_QUERY_RE,
2158 )
2159 ]
2160 seen = {_SENSITIVE_QUERY_RE}
2161 for pipelines in (_LOG_TEXT_PIPELINES, _LOG_MESSAGE_PIPELINES):
2162 for first, second in pipelines:
2163 for steps in (first, second or ()):
2164 for pattern, _replacement, _parts in steps:
2165 if pattern in seen:
2166 continue
2167 seen.add(pattern)
2168 passes.append(
2169 _BarePass(
2170 pattern,
2171 _ASSIGNMENT_REPLACEMENT,
2172 1,
2173 2,
2174 _without_name_boundary(pattern),
2175 )
2176 )
2177 return tuple(passes)
2180_LOG_PATCHER_BARE_PASSES = _log_patcher_bare_passes()
2181_LOG_PATCHER_BARE_GROUPS = {
2182 bare.pattern: (bare.name_group, bare.value_after_group)
2183 for bare in _LOG_PATCHER_BARE_PASSES
2184}
2187def _run_steps(text: str, steps) -> str:
2188 for pattern, replacement, _parts in steps:
2189 text = pattern.sub(replacement, text)
2190 return text
2193def _substitute_tracked(
2194 pattern: "re.Pattern[str]", parts: Any, text: str, origins: list
2195) -> tuple[str, list]:
2196 """``pattern.sub`` on *text*, carrying each character's origin along.
2198 *origins* gives, for every character of *text*, the position in the
2199 original string it came from, or ``-1`` for an inserted one. The result
2200 is the same text ``re.sub`` gives with the matching replacement, and its
2201 origins: a kept group keeps its characters' origins, a literal part is
2202 inserted (``-1``).
2203 """
2204 pieces: list = []
2205 # Accumulate directly to avoid retaining one list per match fragment,
2206 # which can trigger repeated garbage collections on dense credentials.
2207 merged: list = []
2208 position = 0
2209 for match in pattern.finditer(text):
2210 start = match.start()
2211 pieces.append(text[position:start])
2212 merged.extend(origins[position:start])
2213 for part in parts(match) if callable(parts) else parts:
2214 if isinstance(part, str):
2215 pieces.append(part)
2216 merged.extend([-1] * len(part))
2217 continue
2218 group_start, group_end = match.span(part)
2219 if group_start < 0: 2219 ↛ 2220line 2219 didn't jump to line 2220 because the condition on line 2219 was never true
2220 continue
2221 pieces.append(text[group_start:group_end])
2222 merged.extend(origins[group_start:group_end])
2223 position = match.end()
2224 if not pieces:
2225 return text, origins
2226 pieces.append(text[position:])
2227 merged.extend(origins[position:])
2228 return "".join(pieces), merged
2231def _strip_control_chars_tracked(text: str, origins: list) -> tuple[str, list]:
2232 """:func:`strip_control_chars`, carrying origins as
2233 :func:`_substitute_tracked` does."""
2234 kept: list = []
2235 kept_origins: list = []
2236 position = 0
2237 for match in _UNSAFE_CHAR_RE.finditer(text):
2238 kept.append(text[position : match.start()])
2239 kept_origins.extend(origins[position : match.start()])
2240 position = match.end()
2241 if not kept:
2242 return text, origins
2243 kept.append(text[position:])
2244 kept_origins.extend(origins[position:])
2245 return "".join(kept), kept_origins
2248def _redact_bounded_log_text(text: str) -> str:
2249 """:func:`redact_log_message`'s pass, on text already within the bound.
2251 Runs the pipelines of :data:`_LOG_MESSAGE_PIPELINES` and shows nothing
2252 that any of them leaves out (see :func:`_redact_by_pipelines`).
2253 """
2254 return _redact_by_pipelines(text, _LOG_MESSAGE_PIPELINES)
2257def _redact_log_text(value: str) -> str:
2258 """Redact, strip control characters, then redact again.
2260 The first pass sees the original separators, so ``failure\ntoken=x`` does
2261 not glue into an unrecognisable ``failuretoken=x``; the second catches a
2262 name that only becomes recognisable once invisible characters are gone
2263 (``api_<U+200B>key=x``). Used for the message and every ``extra`` string.
2264 The length bound is applied once, up front, so the second pass never
2265 re-truncates the first pass's output.
2267 Several pipelines run these passes, differing only in how they redact
2268 names (see :data:`_LOG_TEXT_PIPELINES`), because no single order is
2269 safe on its own: a replacement made in the first pass changes how the
2270 second pass, on the stripped text, splits the text around it, and a
2271 value left for the second pass can be missed there
2272 (``-\nDB_PASSWORD=x`` is stripped to ``-DB_PASSWORD=x``, a name after
2273 ``-``). The result shows nothing that any pipeline's result leaves
2274 out (see :func:`_redact_by_pipelines`).
2275 """
2276 value = _plain_text(value)
2277 if not value:
2278 return _exact_str(value)
2279 head, marker = _truncate_for_log_redaction(value)
2280 return (
2281 _scrub_to_fixed_point(
2282 lambda text: _redact_by_pipelines(text, _LOG_TEXT_PIPELINES), head
2283 )
2284 + marker
2285 )
2288# A credential name directly followed by ``\s*=``. Every name step's
2289# pattern (:data:`_BARE_SENSITIVE_ASSIGNMENT_RE`,
2290# :data:`_SENSITIVE_ASSIGNMENT_RE`, :data:`_GLUED_TAIL_RE`,
2291# :data:`_LEFTMOST_SENSITIVE_ASSIGNMENT_RE`,
2292# :data:`_START_ORDER_SENSITIVE_ASSIGNMENT_RE`,
2293# :data:`_FIRST_PASS_SENSITIVE_ASSIGNMENT_RE`) ends its name with an
2294# alternative of :data:`_LOG_SENSITIVE_NAME_PATTERN` followed by
2295# ``\s*=``, under the same ``(?i)``, so a text this does not match is
2296# left unchanged by every name step.
2297_NAME_ASSIGNMENT_HINT_RE = re.compile(
2298 "".join((r"(?i)(?:", _LOG_SENSITIVE_NAME_PATTERN, r")\s*="))
2299)
2302def _has_name_assignment_hint(text: str) -> bool:
2303 # No ``=`` at all is the common case, and a substring test is far
2304 # cheaper than the case-insensitive scan over every name.
2305 return "=" in text and _NAME_ASSIGNMENT_HINT_RE.search(text) is not None
2308def _result_without_name_assignments(
2309 shaped: str, pipelines: tuple
2310) -> Optional[str]:
2311 """What every pipeline gives when no name step can match, else ``None``.
2313 *shaped* is the text after :data:`_SHAPE_STEPS`, where every first
2314 pass starts. When neither it nor (for pipelines with a second pass)
2315 the second pass's text after its shape steps holds a credential name
2316 followed by ``=`` (:data:`_NAME_ASSIGNMENT_HINT_RE`), every name step
2317 is a no-op and every pipeline gives the same text, which is returned
2318 -- the text :func:`_redact_by_pipelines` would return, without running
2319 the name steps of each pipeline (most log records). ``None`` means the
2320 pipelines must run.
2321 """
2322 if _has_name_assignment_hint(shaped):
2323 return None
2324 second_passes = {second is None for _first, second in pipelines}
2325 if second_passes == {True}:
2326 return shaped
2327 if second_passes != {False}: 2327 ↛ 2328line 2327 didn't jump to line 2328 because the condition on line 2327 was never true
2328 return None
2329 second_text = _run_steps(strip_control_chars(shaped), _SHAPE_STEPS)
2330 if _has_name_assignment_hint(second_text):
2331 return None
2332 return second_text
2335def _redact_by_pipelines(head: str, pipelines: tuple) -> str:
2336 """*head* as every pipeline of *pipelines* redacts it, together.
2338 Runs them untracked first. When the text holds no credential name
2339 followed by ``=`` (before or after control characters are stripped),
2340 no name step can match and the shape-redacted text is returned
2341 directly (see :func:`_result_without_name_assignments`). In the usual
2342 remaining case they all give the same text, which is returned;
2343 otherwise :func:`_redact_log_text_union` reruns them with origins
2344 tracked.
2345 """
2346 run = _PipelineRun(head, tracked=False, pipelines=pipelines)
2347 without_names = _result_without_name_assignments(run.shaped, pipelines)
2348 if without_names is not None:
2349 return without_names
2350 indexes = range(len(pipelines))
2351 if len({run.first_pass(index) for index in indexes}) == 1:
2352 # The first passes agree, so the pipelines differ at most in the
2353 # name steps of the second.
2354 results = {run.result(index)[0] for index in indexes}
2355 if len(results) == 1:
2356 return results.pop()
2357 return _redact_log_text_union(head, run.unmatched, pipelines)
2360class _PipelineRun:
2361 """The pipelines of *pipelines* (by default :data:`_LOG_TEXT_PIPELINES`)
2362 on one text.
2364 Each pipeline is a pair: the name steps of its first pass and those of
2365 its second, or ``None`` for a pipeline with no second pass (no control
2366 characters stripped). Steps several pipelines share on the same text
2367 (the shape steps of the first pass, the bare names, a second pass
2368 reached from the same first pass) run once. *tracked* carries each
2369 character's origin along (see :func:`_substitute_tracked`); otherwise
2370 origins are ``None``. An untracked run records each step that matched
2371 nothing on a text (:attr:`unmatched`); a tracked run given that set
2372 skips those steps, which leave both the text and its origins unchanged.
2373 """
2375 def __init__(
2376 self,
2377 head: str,
2378 *,
2379 tracked: bool,
2380 unmatched: Optional[set] = None,
2381 pipelines: Optional[tuple] = None,
2382 credential_spans: Optional[list] = None,
2383 origins: Optional[list] = None,
2384 ) -> None:
2385 self._tracked = tracked
2386 self._pipelines = (
2387 _LOG_TEXT_PIPELINES if pipelines is None else pipelines
2388 )
2389 self._memo: dict = {}
2390 self.unmatched: set = set() if unmatched is None else unmatched
2391 self._credential_spans = credential_spans
2392 if tracked and origins is None:
2393 origins = list(range(len(head)))
2394 self._shared = self._run((head, origins), _SHAPE_STEPS)
2396 @property
2397 def shaped(self) -> str:
2398 """The text after :data:`_SHAPE_STEPS`, where every first pass
2399 starts."""
2400 return self._shared[0]
2402 def _cached(self, key: tuple, state: tuple, compute) -> tuple:
2403 # Keyed by text; reused only for equal origins, so two pipelines
2404 # that reach the same text from different characters stay apart.
2405 entries = self._memo.setdefault(key, [])
2406 for origins, result in entries:
2407 if origins is state[1] or origins == state[1]:
2408 return result
2409 result = compute()
2410 entries.append((state[1], result))
2411 return result
2413 def _substitute(self, step: tuple, state: tuple) -> tuple:
2414 pattern, replacement, parts = step
2415 if self._tracked:
2416 if (state[0], step) in self.unmatched:
2417 return state
2418 if self._credential_spans is not None:
2419 self._collect_credential_spans(step, state)
2420 return _substitute_tracked(pattern, parts, *state)
2421 text, count = pattern.subn(replacement, state[0])
2422 if not count:
2423 self.unmatched.add((state[0], step))
2424 return text, None
2426 def _collect_credential_spans(self, step: tuple, state: tuple) -> None:
2427 collected = self._credential_spans
2428 if collected is None: 2428 ↛ 2429line 2428 didn't jump to line 2429 because the condition on line 2428 was never true
2429 return
2430 pattern, replacement, _parts = step
2431 groups = _LOG_PATCHER_BARE_GROUPS.get(pattern)
2432 if groups is None:
2433 return
2434 name_group, value_after = groups
2435 text, origins = state
2436 spans = []
2437 for match in pattern.finditer(text):
2438 rendered = (
2439 replacement(match)
2440 if callable(replacement)
2441 else match.expand(replacement)
2442 )
2443 if rendered == match.group():
2444 continue
2445 name = [
2446 offset
2447 for offset in origins[
2448 match.start(name_group) : match.end(value_after)
2449 ]
2450 if offset >= 0
2451 ]
2452 value = [
2453 offset
2454 for offset in origins[match.end(value_after) : match.end()]
2455 if offset >= 0
2456 ]
2457 if name and value: 2457 ↛ 2437line 2457 didn't jump to line 2437 because the condition on line 2457 was always true
2458 spans.append((min(name), min(value), max(value) + 1))
2459 if spans:
2460 collected.append(spans)
2462 def _run(self, state: tuple, steps: tuple) -> tuple:
2463 for step in steps:
2464 # Reuse identical steps reached through different pipelines.
2465 # _cached also checks origins before reusing a tracked result.
2466 state = self._cached(
2467 (state[0], step),
2468 state,
2469 lambda: self._substitute(step, state),
2470 )
2471 return state
2473 def first_pass(self, index: int) -> tuple:
2474 return self._run(self._shared, self._pipelines[index][0])
2476 def result(self, index: int) -> tuple:
2477 state = self.first_pass(index)
2478 second = self._pipelines[index][1]
2479 if second is None:
2480 return state
2481 if self._tracked:
2482 stripped = self._cached(
2483 (state[0], "strip"),
2484 state,
2485 lambda: _strip_control_chars_tracked(*state),
2486 )
2487 else:
2488 stripped = (strip_control_chars(state[0]), None)
2489 return self._run(stripped, _SHAPE_STEPS + second)
2492_LOG_TEXT_PIPELINE_INDEXES = range(len(_LOG_TEXT_PIPELINES))
2495def _pipeline_credential_spans(text: str) -> list:
2496 """Map later name/value readings back to the original input."""
2497 spans: list = []
2498 run = _PipelineRun(text, tracked=True, credential_spans=spans)
2499 for index in _LOG_TEXT_PIPELINE_INDEXES:
2500 run.result(index)
2501 # A plain query replacement can expose a name that the original URL
2502 # made part of another value. Cover the later log-pipeline readings
2503 # after that replacement too, before extending across line breaks or
2504 # quoted values would remove the newly exposed name.
2505 pieces: list[str] = []
2506 origins: list[int] = []
2507 position = 0
2508 for match in _URL_QUERY_STRING_RE.finditer(text):
2509 pieces.append(text[position : match.start()])
2510 origins.extend(range(position, match.start()))
2511 replacement = _restore_query_wrapper(match)
2512 pieces.append(replacement)
2513 origins.extend(range(match.start(), match.start(2)))
2514 origins.extend([-1] * len(_URL_QUERY_MARKER))
2515 if len(replacement) > len(match.group(1)) + 1 + len(_URL_QUERY_MARKER):
2516 origins.append(match.end() - 1)
2517 position = match.end()
2518 if pieces:
2519 pieces.append(text[position:])
2520 origins.extend(range(position, len(text)))
2521 projected = "".join(pieces)
2522 if projected != text:
2523 run = _PipelineRun(
2524 projected,
2525 tracked=True,
2526 credential_spans=spans,
2527 origins=origins,
2528 )
2529 for index in _LOG_TEXT_PIPELINE_INDEXES:
2530 run.result(index)
2531 return spans
2534def _redact_log_text_union(
2535 head: str, unmatched: set, pipelines: Optional[tuple] = None
2536) -> str:
2537 """Every pipeline of *pipelines* (by default :data:`_LOG_TEXT_PIPELINES`),
2538 with origins tracked.
2540 Each character of a pipeline's result is followed back to the position
2541 of *head* it came from (see :func:`_substitute_tracked`). A position
2542 that some pipeline's result no longer holds was redacted (or stripped)
2543 there; the first pipeline's result is returned with every run of such
2544 characters replaced by one ``[REDACTED]``. A pipeline whose result is
2545 the same text as the first's adds nothing and is skipped. *unmatched*
2546 is :attr:`_PipelineRun.unmatched` of an untracked run on *head*.
2547 """
2548 run = _PipelineRun(
2549 head, tracked=True, unmatched=unmatched, pipelines=pipelines
2550 )
2551 kept_text, kept_origins = run.result(0)
2552 removed = bytearray(len(head))
2553 for index in range(1, len(run._pipelines)):
2554 text, origins = run.result(index)
2555 if text == kept_text:
2556 # Prints exactly what the kept text prints.
2557 continue
2558 survived = bytearray(len(head))
2559 for origin in origins:
2560 if origin >= 0:
2561 survived[origin] = 1
2562 for origin in kept_origins:
2563 if origin >= 0 and not survived[origin]:
2564 removed[origin] = 1
2565 out: list = []
2566 in_run = False
2567 width = len(REDACTION_TEXT)
2568 for position, (character, origin) in enumerate(
2569 zip(kept_text, kept_origins)
2570 ):
2571 if origin >= 0 and removed[origin]:
2572 # A run next to a replacement the kept pipeline made already
2573 # reads ``[REDACTED]``; do not double it.
2574 if not in_run and not "".join(out[-width:]).endswith(
2575 REDACTION_TEXT
2576 ):
2577 out.append(REDACTION_TEXT)
2578 in_run = True
2579 continue
2580 if (
2581 in_run
2582 and out[-1] is REDACTION_TEXT
2583 and kept_text.startswith(REDACTION_TEXT, position)
2584 and max(kept_origins[position : position + width]) < 0
2585 ):
2586 out.pop()
2587 in_run = False
2588 out.append(character)
2589 return "".join(out)
2592def _type_is_one_of(kind: Any, types: tuple) -> bool:
2593 """True when *kind* is one of *types* (by identity).
2595 ``kind in types`` compares with ``==``, which a metaclass ``__eq__`` of
2596 an application class can answer ``True`` for ``str`` or ``list``.
2597 """
2598 return any(kind is candidate for candidate in types)
2601# Credential shapes the log-record path redacts on top of the passes above.
2602# They run ONLY as the last step of :func:`redact_log_record`'s message and
2603# ``extra`` string redaction (:func:`_redact_log_record_text`), its rendered
2604# exception fields (:func:`_redact_exception_text`), and the exception prefix
2605# of a database log row (``utilities.log_utils._exception_context``), whose
2606# output goes straight to the sinks: no pass of
2607# ``sanitize_error_message``, ``scrub_error``, the client helpers or
2608# :func:`redact_log_message` ever runs over what these patterns write. Those
2609# functions do not apply them, so their output is what it was before these
2610# patterns existed. These patterns only replace part of the text they are
2611# given with a ``[REDACTED]`` marker; whatever the earlier passes redacted
2612# stays redacted, so where they run they can hide more, never less.
2613#
2614# Their value boundaries are conservative, and what may follow a value is an
2615# allowlist. A value stops before a credential label (``_VALUE_STOP``; that
2616# label is then matched on its own) and before a URL scheme, and it is
2617# redacted only when what follows it ends a value in its syntax: one of the
2618# structural followers listed below for its kind, after optional whitespace
2619# that is not a line break, or a line break, or the end of the text
2620# (``_cuts_a_value``, ``_API_KEY_FOLLOWER_RE``, ``_DSN_HOST_RE``). Anything
2621# else -- a word after a space, another quote, a backslash, an opening
2622# bracket, or a symbol a secret may contain (``&``, ``!``, ``$``, ``/``,
2623# ``.``, ``@`` ...) -- means the value may go on past the match, so the
2624# text is left exactly as the earlier passes left it: replacing only part
2625# of a value would print the rest of it behind a marker that makes the
2626# line look redacted. A
2627# quoted value holding the other kind of quote, an unquoted value that
2628# starts with a string prefix before a quote (``_UNQUOTED_PREFIX_STOP``) and
2629# one that leaves a bracket it opened unclosed are left as they were too.
2630#
2631# What no allowlist can tell apart: a secret written raw into a quoted or
2632# unquoted slot (JSON and a repr escape a quote inside a string; a
2633# hand-built ``f"{{'password': '{pw}'}}"`` does not) that itself holds a
2634# quote or a value end followed by a structural follower. ``'ab',cd'`` or
2635# ``{'password': ab,cd}`` has exactly the shape of the value ``'ab'`` (or
2636# ``ab``) followed by another item, so the part after the follower prints
2637# behind the marker. The same holds for a DSN password that holds an '@'
2638# followed by what reads as a host and its end (``_DSN_HOST_RE``).
2639#
2640# The structural followers, by the kind of value they follow. After a
2641# quoted value's closing quote (a JSON or repr string, a quoted parameter):
2642# ``,`` the next item of an object, a list, a call or a parameter list;
2643# ``}``, ``]`` and ``)`` the end of an object, a list, a call or a tuple;
2644# ``:`` the separator after an item in a ``name: value`` listing
2645# (``f"{key!r}: {value!r}: {reason}"``); ``;`` the next parameter or cookie
2646# attribute (``password="..."; Path=/``); ``&`` the next query or form
2647# parameter; ``#`` a URL fragment or a trailing comment.
2648_QUOTED_VALUE_FOLLOWERS = ",}]):;&#"
2649# After an unquoted value of a quoted key (JSON or a repr, where an unquoted
2650# value is a number, a literal, a name or a call): ``,`` the next item, and
2651# ``}`` or ``]`` the end of the object or list. Nothing else ends such a
2652# value: ``&`` and ``#`` never do in JSON or a repr, the run takes ``;``,
2653# ``:`` and ``)`` in (a call ends at its own ``)``), and a word after a
2654# space means the value may be several words (``Basic dXNl...``,
2655# ``correct horse battery staple``).
2656_KEY_VALUE_FOLLOWERS = ",}]"
2657# After an unquoted ``;``/``,`` parameter value (matrix parameters, joined
2658# lists, call arguments): ``,`` and ``;`` the next parameter, and ``)``,
2659# ``]`` and ``}`` the end of the call, list or object the parameters are
2660# in. ``&`` and ``#`` belong to ``?``/``&`` query syntax (its separator
2661# and the fragment), not to these parameters, so a value that stops at one
2662# may go on past it (``;key=ab&cd``), and a word after a space means the
2663# value may be several words.
2664_PARAM_VALUE_FOLLOWERS = ",;)]}"
2665# After an ``Api-Key:`` header value (a run of the header pattern's token
2666# class, below): ``,``, ``;``, ``)``, ``]`` and ``}`` end the list, call or
2667# object the header sits in, and a word after a space means the value may
2668# be several words, as above. A quote that closes the string the header
2669# sits in counts too, when one of ``_QUOTED_VALUE_FOLLOWERS``, a line break
2670# or the end of the text follows it (``['Api-Key: ...', ...]``), and so
2671# does ``://`` straight after the run: a URL glued to the key, whose scheme
2672# letters the run took in.
2673_API_KEY_VALUE_FOLLOWERS = ",;)]}"
2676def _followers_pattern(followers: str) -> str:
2677 """Matches at the end of a value only when the value ends there:
2678 optional whitespace other than a line break, then one of *followers*,
2679 a line break (``\\n``, ``\\r``) or the end of the text."""
2680 return r"[^\S\r\n]*+(?:[" + re.escape(followers) + r"\r\n]|\Z)"
2683_QUOTED_VALUE_FOLLOWER_RE = re.compile(
2684 _followers_pattern(_QUOTED_VALUE_FOLLOWERS)
2685)
2686_KEY_VALUE_FOLLOWER_RE = re.compile(_followers_pattern(_KEY_VALUE_FOLLOWERS))
2687_PARAM_VALUE_FOLLOWER_RE = re.compile(
2688 _followers_pattern(_PARAM_VALUE_FOLLOWERS)
2689)
2690_API_KEY_FOLLOWER_RE = re.compile(
2691 "://|"
2692 + _followers_pattern(_API_KEY_VALUE_FOLLOWERS)
2693 + r"|[^\S\r\n]*+[\"'](?="
2694 + _followers_pattern(_QUOTED_VALUE_FOLLOWERS)
2695 + ")"
2696)
2697# After the '@' that ends a would-be DSN's userinfo: a host (a name or an
2698# IPv4 address, an IPv6 literal in brackets, or a ``${VAR}`` placeholder)
2699# followed by what ends a host: whitespace, ``:`` before a port, ``/``,
2700# ``?``, ``#``, a quote, ``<``, ``>``, ``,``, ``;``, ``)``, ``]``, ``}``, a
2701# backslash or the end of the text. Anything else after the '@' (another
2702# '@', a symbol, a space) means the password may hold an unescaped '@' and
2703# go on past it.
2704_DSN_HOST_RE = re.compile(
2705 r"(?:[A-Za-z0-9][A-Za-z0-9.\-_~%]*+|\[[0-9A-Za-z:.%]++\]|\$\{\w++\})"
2706 r"(?=[\s:/?#\"'<>,;)\]}\\]|\Z)"
2707)
2708# Credential parameter names for the ``;``/``,`` parameter pattern below:
2709# the same names the ``?``/``&`` query-parameter pattern above redacts.
2710_CREDENTIAL_PARAM_NAMES = (
2711 r"api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|"
2712 r"auth[_-]?token|session[_-]?token|secret[_-]?key|bearer[_-]?token|"
2713 r"subscription[_-]?key|client[_-]?secret|api[_-]?secret|"
2714 r"app[_-]?secret|private[_-]?key|"
2715 r"key|token|secret|password|passwd|pwd"
2716)
2717# A credential label: a ``name=`` that ``redact_sensitive_assignments``
2718# redacts (a ``LOG_SENSITIVE_KEYS`` name, optional whitespace before the
2719# ``=``) or that the parameter pattern below redacts
2720# (``_CREDENTIAL_PARAM_NAMES``).
2721_CREDENTIAL_LABEL = (
2722 r"(?:(?i:" + _LOG_SENSITIVE_NAME_PATTERN + r")\s*="
2723 r"|(?i:" + _CREDENTIAL_PARAM_NAMES + r")=)"
2724)
2725# Checked before each character of a value matched below. A credential
2726# label is found only after a character outside ``[\w.-]`` (the assignment
2727# pass's left boundary; the ``,``/``;`` anchors of the parameter pattern),
2728# so the value stops before such a character when a label follows it: the
2729# label and its anchor stay as they were, legible, and the label's own
2730# value is matched separately.
2731_LABEL_STOP = "".join((r"(?![^\w.\-]", _CREDENTIAL_LABEL, r")"))
2732# At the first character of a value: no label there either.
2733_NO_LABEL = "".join((r"(?!", _CREDENTIAL_LABEL, r")"))
2734# Also checked before each character: a value stops before a URL scheme
2735# that starts a run of scheme characters (a letter, then letters, digits,
2736# ``+``, ``.`` or ``-`` reaching ``://`` within 32 characters), so a URL
2737# after a value stays legible; the URL-credential patterns have already
2738# redacted its userinfo.
2739_SCHEME_STOP = r"(?!(?<![A-Za-z0-9+.\-])[A-Za-z][A-Za-z0-9+.\-]{0,31}://)"
2740_VALUE_STOP = _LABEL_STOP + _SCHEME_STOP
2741# A name the assignment pass redacts, at the start of its ``name=``.
2742_ASSIGNMENT_NAME = "".join((r"(?:(?i:", _LOG_SENSITIVE_NAME_PATTERN, r")\s*=)"))
2743# Quoted keys (JSON, Python repr) name a credential with the names an
2744# ``extra`` key is redacted under: ``LOG_SENSITIVE_KEYS``, through
2745# ``_LOG_SENSITIVE_NAME_PATTERN`` (``_`` also matching ``-`` or nothing), so
2746# the message text and the ``extra`` keys cannot drift apart. As for an
2747# ``extra`` key (``_is_sensitive_log_key``: the last dotted segment, equal
2748# to a name or ending in ``_`` and the name), the name may follow any
2749# prefix that ends in a ``_``, ``.`` or ``-`` (``_QUOTED_KEY_PREFIX``):
2750# ``"_api_key"``, ``"db__password"``, ``"csrf_token"``, the request-header
2751# dicts the search engines send (``'X-API-KEY'``, ``'x-api-key'``,
2752# ``'X-Api-Key'``, ``'X-Subscription-Token'``), settings dumps
2753# (``"openai_api_key"``, ``"Ocp-Apim-Subscription-Key"``), the project's
2754# dotted settings keys (``"llm.openai.api_key"``) and its environment
2755# variables (``"LDR_SEARCH_ENGINE_WEB_TAVILY_API_KEY"``). A ``-`` before the
2756# name counts too, which an ``extra`` key does not need. The name must end
2757# the key, so ``"secret_name"`` or ``"password_hint"`` is not a credential
2758# key; a name glued to a camelCase prefix (``"dbPassword"``) is not one
2759# either, as for an ``extra`` key. Any key that ends in one is, so
2760# ``"show_password"`` or ``"use_secret"`` is redacted unless its value is a
2761# bare literal (over-redaction only); ``"requires_api_key"``, which
2762# ``_is_sensitive_log_key`` exempts, is redacted too when its value is not
2763# a bare literal.
2764#
2765# The cost stays linear in the key: the prefix cannot contain a quote, a
2766# backslash or a newline, so the scan from one opening quote ends before
2767# the next one starts; the prefix is lazy, so a name is tried once at each
2768# position, and the atomic group commits to the first position whose name
2769# reaches the closing quote (every decomposition ends at that same quote),
2770# so the value is matched once per key.
2771_QUOTED_CREDENTIAL_KEY_NAMES = _LOG_SENSITIVE_NAME_PATTERN
2772_QUOTED_KEY_PREFIX = r"(?:[^\"'\n\\]*?[_.\-])?"
2773# An unquoted value that is a whole JSON/Python literal (``true``,
2774# ``False``, ``None``, ``null``) is not a secret; with prefixed names
2775# (``"requires_api_key": True``) it is common, so it is left legible.
2776_UNQUOTED_LITERAL_STOP = r"(?!(?:true|false|none|null)(?![^\s,}\]&#\"'\\]))"
2777# A label inside a would-be DSN password, with the assignment pass's own
2778# left boundary.
2779_LABEL_IN_USERINFO_RE = re.compile(r"(?<![\w.?&-])" + _CREDENTIAL_LABEL)
2781# A quoted value for the parameter and quoted-key patterns below: to its
2782# closing quote (or the end of the text), with no credential label or URL
2783# scheme inside, and no quote character of the other kind (escaped or
2784# not). ``_VALUE_STOP`` is checked before every character, the one after a
2785# backslash included: a backslash escape is two characters, and checking
2786# only before the backslash would let an escaped separator (a backslash and
2787# a comma) hide a quoted label's ``password =`` after it, so the value would
2788# run on to that label's own opening quote and take the label with it.
2789_QUOTED_VALUE = "".join(
2790 (
2791 r"\"",
2792 _NO_LABEL,
2793 r"(?:",
2794 _VALUE_STOP,
2795 r"(?:\\",
2796 _VALUE_STOP,
2797 r"[^'\n]|[^\"'\\]))*(?:\"|\\?$)",
2798 r"|'",
2799 _NO_LABEL,
2800 r"(?:",
2801 _VALUE_STOP,
2802 r"(?:\\",
2803 _VALUE_STOP,
2804 r"[^\"\n]|[^'\"\\]))*(?:'|\\?$)",
2805 )
2806)
2807# A quoted value as above, after an optional Python string prefix
2808# (``b'...'``, ``rb"..."``, ``f'...'``), so a bytes value or a bytes-keyed
2809# repr (``{b'password': b'...'}``) is redacted whole.
2810# ``_redacted_value`` keeps the prefix.
2811_PREFIXED_QUOTED_VALUE = "".join(
2812 (r"(?:[bruf]{1,2}(?=[\"']))?(?:", _QUOTED_VALUE, r")")
2813)
2814# At the first character of an unquoted value: not a string prefix before a
2815# quote. Matched as an unquoted run, ``b'secret'`` would stop at the quote,
2816# so the marker would replace only the ``b``; it is matched whole by
2817# ``_PREFIXED_QUOTED_VALUE`` instead, or left as it is when that refuses it.
2818_UNQUOTED_PREFIX_STOP = r"(?![bruf]{1,2}[\"'])"
2819# An unquoted value of a quoted key that is written as an ``Authorization``
2820# header's is: a scheme word (the schemes ``sanitize_error_message``'s
2821# ``Authorization`` pattern names) and one credential token. It is taken as
2822# one value, whatever the key, so the marker replaces both
2823# (``{'Authorization': Basic dXNl...}``). Anything but a structural
2824# follower after the token (a second token, the ``username=...,
2825# response=...`` list of a ``Digest`` value) leaves the text as it was.
2826_SCHEME_CREDENTIAL = (
2827 r"(?:basic|bearer|digest|negotiate|apikey|token)"
2828 r"[^\S\r\n]++[A-Za-z0-9\-._~+/]++=*"
2829)
2830# Brackets an unquoted value may open and the ones that close them.
2831_VALUE_BRACKETS = (("(", ")"), ("[", "]"), ("{", "}"))
2834def _cuts_a_value(
2835 text: str,
2836 start: int,
2837 end: int,
2838 unquoted_followers: "re.Pattern[str]",
2839) -> bool:
2840 """True unless ``text[start:end]``, a value matched below, ends where
2841 its syntax ends a value, so that replacing it cannot print the rest of
2842 the value behind a ``[REDACTED]`` marker that makes the line look
2843 redacted. The callers then return the text unchanged.
2845 An allowlist: the value is taken as whole only when what follows it
2846 matches ``_QUOTED_VALUE_FOLLOWER_RE`` (after a quoted value's closing
2847 quote, or its end when it runs to the end of the text) or
2848 *unquoted_followers* (after an unquoted run): a structural follower for
2849 that kind of value, after optional whitespace that is not a line
2850 break, or a line break, or the end of the text. Any other character
2851 refuses it: a quote (``'''secret'''``, whose first two quotes are an
2852 empty string, or a quote that opens a string belonging to the value,
2853 ``SecretStr('...')``), a backslash, a word after a space, or a symbol a
2854 secret may hold (``'Pa'$$w0rd'``, ``hunter2&rest``). An unquoted value
2855 that leaves a bracket it opened unclosed is not whole either, whatever
2856 follows it: a nested container or a call stops at its first separator
2857 (``(1, '...')``, ``{0: '...'}``, ``Foo(a, '...')``), so the rest of it
2858 is outside the match.
2859 """
2860 value = text[start:end]
2861 if _redacted_value(value) != "[REDACTED]":
2862 # A quoted value, string prefix and all (``b'...'``).
2863 return _QUOTED_VALUE_FOLLOWER_RE.match(text, end) is None
2864 if unquoted_followers.match(text, end) is None:
2865 return True
2866 return any(
2867 value.count(opener) > value.count(closer)
2868 for opener, closer in _VALUE_BRACKETS
2869 )
2872def _redacted_value(value: str) -> str:
2873 """``[REDACTED]`` in place of a value matched by the patterns below.
2875 A quoted value keeps its quotes (its closing quote only when it has
2876 one) and its string prefix (``b'[REDACTED]'``), so the text after it
2877 is still delimited as it was.
2878 """
2879 stripped = value.lstrip("bBrRuUfF")
2880 prefix = value[: len(value) - len(stripped)]
2881 quote = stripped[:1]
2882 if quote not in ('"', "'") or len(prefix) > 2:
2883 return "[REDACTED]"
2884 body = stripped[1:]
2885 closed = False
2886 if body.endswith(quote):
2887 inner = body[:-1]
2888 closed = (len(inner) - len(inner.rstrip("\\"))) % 2 == 0
2889 return prefix + quote + "[REDACTED]" + (quote if closed else "")
2892def _redact_slash_dsn(match: "re.Match[str]") -> str:
2893 """Replacement for the slash-DSN pattern below.
2895 A password run that holds a credential label (``/password=...``,
2896 ``;token=...``) is not userinfo but a path running into that label's
2897 value, up to an '@' inside it. Redacting it would remove the label the
2898 parameter pattern below needs and print the value's tail, so the text
2899 is returned unchanged for it. Checked here rather than inside the
2900 regex: a per-character label lookahead made the capped scan several
2901 times slower.
2903 The '@' is taken as the end of the userinfo only when a host follows
2904 it and ends where a host ends (``_DSN_HOST_RE``). Anything else, such
2905 as a second '@' (``u:pa/ss@word@host``), a symbol or a space after it,
2906 means the password may hold an unescaped '@', so which '@' ends the
2907 userinfo cannot be told: the text is returned unchanged rather than
2908 printing the password's tail behind the marker.
2909 """
2910 if _LABEL_IN_USERINFO_RE.search(match.string, match.start(4), match.end(4)): 2910 ↛ 2911line 2910 didn't jump to line 2911 because the condition on line 2910 was never true
2911 return match.group(0)
2912 if _DSN_HOST_RE.match(match.string, match.end()) is None:
2913 return match.group(0)
2914 return match.group(1) + match.group(2) + "[REDACTED]:[REDACTED]@"
2917def _redact_api_key_header(match: "re.Match[str]") -> str:
2918 """Replacement for the ``Api-Key:`` header pattern below.
2920 The text is returned unchanged unless the value ends where a header
2921 value ends (``_API_KEY_FOLLOWER_RE``).
2922 """
2923 if _API_KEY_FOLLOWER_RE.match(match.string, match.end()) is None:
2924 return match.group(0)
2925 return match.group(1) + "[REDACTED]"
2928def _redact_credential_param(match: "re.Match[str]") -> str:
2929 """Replacement for the ``;``/``,`` credential-parameter pattern below.
2931 The text is returned unchanged unless the matched value ends where a
2932 value ends (:func:`_cuts_a_value`, with ``_PARAM_VALUE_FOLLOWERS`` after
2933 an unquoted value).
2934 """
2935 group = 3 if match.group(3) is not None else 5
2936 if _cuts_a_value(
2937 match.string,
2938 match.start(group),
2939 match.end(group),
2940 _PARAM_VALUE_FOLLOWER_RE,
2941 ):
2942 return match.group(0)
2943 return (
2944 match.group(1)
2945 + (match.group(2) or match.group(4))
2946 + "="
2947 + _redacted_value(match.group(group))
2948 )
2951def _redact_quoted_key(match: "re.Match[str]") -> str:
2952 """Replacement for the quoted-credential-key pattern below.
2954 The text is returned unchanged unless the matched value ends where a
2955 value ends (:func:`_cuts_a_value`, with ``_KEY_VALUE_FOLLOWERS`` after
2956 an unquoted value).
2957 """
2958 if _cuts_a_value(
2959 match.string, match.start(4), match.end(4), _KEY_VALUE_FOLLOWER_RE
2960 ):
2961 return match.group(0)
2962 return (
2963 match.group(1)
2964 + match.group(2)
2965 + match.group(1)
2966 + match.group(3)
2967 + _redacted_value(match.group(4))
2968 )
2971_ADDITIONAL_CREDENTIAL_PATTERNS: list[
2972 tuple[re.Pattern[str], Union[str, Callable[["re.Match[str]"], str]]]
2973] = [
2974 # Slash-bearing DSN passwords — base64/base64-ish generated secrets
2975 # pasted into a connection string routinely contain an unescaped '/',
2976 # which the URL-credentials pattern in ``_CREDENTIAL_PATTERNS`` refuses
2977 # by design. Admitting the slash makes one URL shape look like
2978 # userinfo: ``host:port/path...@...``
2979 # (``https://api.example.com:443/users/bob@example.com``), where the
2980 # "user" is really the host and the "password" a port followed by a
2981 # path. A password that is a run of digits followed by ``/``, ``?`` or
2982 # ``#`` (or that starts with one of them) is that shape, so it is
2983 # refused; the user part also refuses ``[``/``]`` so an IPv6 literal
2984 # host is never read as userinfo. Any host is accepted after the '@'
2985 # (``@postgres:5432``, ``@localhost``, ``@10.0.0.5`` are the usual
2986 # container DSNs). Known gap: a real password that is digits followed
2987 # by '/' is left alone.
2988 #
2989 # Neither group crosses a JSON, list or query boundary: the user refuses
2990 # ``"'<>,`` and ``?#&=;``, the password ``"'<>,`` and ``?#&``. A
2991 # match whose password holds a credential label (``/password=``,
2992 # ``;token=``, ``=api_key=``), or whose '@' is not followed by a host
2993 # and a host's end (``_DSN_HOST_RE``; ``u:pa/ss@word@host``, where the
2994 # password may hold an unescaped '@'), is left alone
2995 # (``_redact_slash_dsn``). Known gap: a slash-bearing password that
2996 # also contains one of those characters is not redacted by this
2997 # pattern, nor one that contains an '@' unless what follows that '@'
2998 # reads as a host and its end (``u:pa/ss@word/x@host``), which cannot
2999 # be told from a DSN followed by more text, so the rest of such a
3000 # password prints behind the marker.
3001 #
3002 # Both groups are length-capped: an uncapped password class that may
3003 # run through '/' made matching quadratic on a whitespace-free run of
3004 # ``scheme://:`` with no '@' (seconds of GIL-holding CPU at ~30k
3005 # chars). The caps bound the work per start position, which keeps the
3006 # scan linear; a longer user or password is not redacted by this
3007 # pattern.
3008 #
3009 # Each ``://`` is also tried once, not once per scheme start: a run of
3010 # scheme characters is entered only at its first character (the
3011 # lookbehind), a lazy prefix (group 1, put back by the replacement)
3012 # skips to the leftmost position where a scheme reaches the ``://`` --
3013 # where an unanchored search would have started -- and the atomic group
3014 # commits to that scheme. Every later start before the same ``://``
3015 # would retry the identical user/password tail, so dropping them changes
3016 # no result; without this, a 32-letter scheme repeated with ``://:``
3017 # ran up to 31 capped tails per ``://`` (about 6x the cost of the
3018 # URL-credentials pattern on the same run). The user and password
3019 # quantifiers are possessive: the character that must follow each is
3020 # outside its class, so giving characters back can never succeed.
3021 (
3022 re.compile(
3023 r"(?<![A-Za-z0-9+.\-])"
3024 r"(?>([A-Za-z0-9+.\-]*?)([A-Za-z][A-Za-z0-9+.\-]{1,31}://))"
3025 r"([^:\s/@\[\]\"'<>,?#&=;]{0,128}+):(?![0-9]*[/?#])"
3026 r"([^@\s\"'<>,?&#]{1,256}+)@"
3027 ),
3028 _redact_slash_dsn,
3029 ),
3030 # API-key header spellings the ``x-api-key`` pattern in
3031 # ``_CREDENTIAL_PATTERNS`` misses — the unprefixed ``Api-Key:`` form is
3032 # what Anthropic, Azure Cognitive Services and several search APIs
3033 # send, plus ``x-api_key``/``x-apikey``. Like the ``x-api-key``
3034 # pattern, the label is a strong anchor, so any value of >=16
3035 # characters is redacted regardless of shape; short values like
3036 # "invalid"/"missing" stay intact. The unprefixed forms match only the
3037 # ``:`` header-dump shape — an ``=`` assignment is prose (``api_key=...``
3038 # mid-sentence is not redacted by this pattern). The value stops where
3039 # a credential label begins (``KEY/client_secret=...``) or a URL scheme
3040 # starts a run (``api_key: postgresql+asyncpg://...``), and is not
3041 # redacted when fewer than 16 characters come before that. It is
3042 # redacted only when it ends where a value ends
3043 # (``_API_KEY_FOLLOWER_RE``): one of ``_API_KEY_VALUE_FOLLOWERS``, a
3044 # line break or the end of the text, after optional whitespace; a
3045 # quote that closes the string the header sits in; or ``://`` glued to
3046 # it. A key followed by a word after a space, by ``&``, ``!``, ``:``,
3047 # a ``/`` before a label or any other character its class lacks is
3048 # left as the earlier passes left it.
3049 #
3050 # Known gap: where a key glued straight to a scheme (``KEYhttps://``)
3051 # ends and the scheme begins cannot be told apart. When the run of
3052 # scheme characters starts at a letter within 32 characters of
3053 # ``://``, the value stops there, so the key prints, all of it, since
3054 # what follows the stop is a letter and not a header value's end. A run
3055 # that starts with a digit or is longer than that is taken whole,
3056 # scheme included.
3057 (
3058 re.compile(
3059 r"(?i)((?:x-api[_-]?key\s*[:=]|api[_-]?key\s*:)\s*)"
3060 + _NO_LABEL
3061 + r"(?:"
3062 + _VALUE_STOP
3063 + r"[A-Za-z0-9\-._~+/]){16,}+=*"
3064 ),
3065 _redact_api_key_header,
3066 ),
3067 # Credential parameters after ``;`` (matrix parameters, ``;token=...``)
3068 # or ``,`` (joined lists, ``,password=...``), with the ``?``/``&``
3069 # parameter pattern's names. The value is a quoted string, optionally
3070 # string-prefixed (``_PREFIXED_QUOTED_VALUE``), or an unquoted run that
3071 # does not start with a string prefix before a quote
3072 # (``_UNQUOTED_PREFIX_STOP``). For a name the assignment
3073 # pass also redacts (``password``, ``token``, ``api_key``, ...) an
3074 # unquoted run must reach whitespace (or the end of the text) without a
3075 # ``&``, ``#``, quote, backslash, label or scheme on the way; any other
3076 # value of such a name is left as the earlier passes left it. Any other
3077 # name (``key``) takes a run that stops at whitespace, ``&``, ``#``, a
3078 # quote, a backslash or a ``,``/``;`` starting a credential label
3079 # (``,key=abc,password="example secret"`` redacts both). Either is
3080 # redacted only when what follows it ends a value (``_cuts_a_value``):
3081 # one of ``_QUOTED_VALUE_FOLLOWERS`` after a quoted value, one of
3082 # ``_PARAM_VALUE_FOLLOWERS`` after an unquoted one, after optional
3083 # whitespace, or a line break or the end of the text; so a run that
3084 # stops at a space before a word (``;key=abc def``), at a quote, at a
3085 # backslash, or at a label after any character but ``,``/``;`` is not
3086 # redacted, nor an unquoted value that leaves a bracket it opened
3087 # unclosed (``,key=[b'...']``). Such a value is left as the earlier
3088 # passes left it.
3089 (
3090 re.compile(
3091 r"(?i)([,;])(?:"
3092 r"(?=" + _ASSIGNMENT_NAME + r")(" + _CREDENTIAL_PARAM_NAMES + r")="
3093 r"("
3094 + _PREFIXED_QUOTED_VALUE
3095 + r"|(?![\"'])"
3096 + _UNQUOTED_PREFIX_STOP
3097 + _NO_LABEL
3098 + r"(?:"
3099 + _VALUE_STOP
3100 + r"[^\s&#\"'\\])++(?!\S))"
3101 r"|(?!" + _ASSIGNMENT_NAME + r")(" + _CREDENTIAL_PARAM_NAMES + r")="
3102 r"("
3103 + _PREFIXED_QUOTED_VALUE
3104 + r"|"
3105 + _UNQUOTED_PREFIX_STOP
3106 + _NO_LABEL
3107 + r"(?:[,;]"
3108 + _NO_LABEL
3109 + r"|"
3110 + _VALUE_STOP
3111 + r"[^\s,;&#\"'\\])++)"
3112 r")"
3113 ),
3114 _redact_credential_param,
3115 ),
3116 # Quoted credential keys in JSON or a Python repr ("password": "...",
3117 # {'api-key': '...'}, {'X-API-KEY': '...'}, "llm.openai.api_key": ...).
3118 # The value is a quoted string, optionally string-prefixed
3119 # (``_PREFIXED_QUOTED_VALUE``: ``b'...'``), a scheme word and one
3120 # credential token (``_SCHEME_CREDENTIAL``: ``Basic dXNl...``), or an
3121 # unquoted run (to whitespace, ``,``, ``}``, ``]``, ``&``, ``#``, a
3122 # quote, a backslash, a credential label or a scheme); an unquoted run
3123 # never starts with a quote or ``[``, so an already-redacted
3124 # ``[REDACTED]`` value is left as it is, nor with a string prefix before
3125 # a quote (``_UNQUOTED_PREFIX_STOP``), and is never a bare
3126 # ``true``/``false``/``None``/``null``. It is redacted only when what
3127 # follows it ends a value (``_cuts_a_value``): one of
3128 # ``_QUOTED_VALUE_FOLLOWERS`` after a quoted value, one of
3129 # ``_KEY_VALUE_FOLLOWERS`` (``,``, ``}``, ``]``) after an unquoted one,
3130 # after optional whitespace, or a line break or the end of the text; an
3131 # unquoted value that leaves a bracket it opened unclosed (``(b'...',)``,
3132 # ``bytearray(b'...')``, ``{0: '...'}``) is not whole either. Any other
3133 # value is left as the earlier passes left it: ``'''...'''``,
3134 # ``'Pa'$$w0rd'``, ``hunter2&rest``, ``correct horse battery staple``,
3135 # ``ApiKey(key=...)``.
3136 (
3137 re.compile(
3138 r"(?i)([\"'])(?>("
3139 + _QUOTED_KEY_PREFIX
3140 + r"(?:"
3141 + _QUOTED_CREDENTIAL_KEY_NAMES
3142 + r"))\1)(\s*:\s*)"
3143 r"("
3144 + _PREFIXED_QUOTED_VALUE
3145 + r"|"
3146 + _SCHEME_CREDENTIAL
3147 + r"|(?![\"'\[])"
3148 + _UNQUOTED_PREFIX_STOP
3149 + _UNQUOTED_LITERAL_STOP
3150 + _NO_LABEL
3151 + r"(?:"
3152 + _VALUE_STOP
3153 + r"[^\s,}\]&#\"'\\])+)"
3154 ),
3155 _redact_quoted_key,
3156 ),
3157]
3160def _redact_additional_shapes(message: str) -> str:
3161 """Apply ``_ADDITIONAL_CREDENTIAL_PATTERNS`` in order, as plain substitutions.
3163 Used as the final redaction step for record text, rendered exception
3164 fields, and the database log row's exception prefix. Callers bound the
3165 text before applying these patterns.
3166 """
3167 if not message:
3168 return message
3169 for pattern, replacement in _ADDITIONAL_CREDENTIAL_PATTERNS:
3170 message = pattern.sub(replacement, message)
3171 return message
3174def _redact_log_record_text(value: str) -> str:
3175 """:func:`_redact_log_text`, then ``_ADDITIONAL_CREDENTIAL_PATTERNS``.
3177 For the log record's message and its ``extra`` string leaves, whose
3178 redacted text goes straight to the sinks. The text is cut to the bound
3179 first, :func:`_redact_log_text` redacts that head, and the added
3180 patterns run on its output, so they see what its passes left and can
3181 only replace more of it; the omission marker is appended after them.
3182 """
3183 value = _plain_text(value)
3184 if not value:
3185 return _exact_str(value)
3186 # Cut once here so the added patterns see a bounded string and the
3187 # omission marker stays outside their reach. ``head`` is within the
3188 # bound, so ``_redact_log_text`` cuts nothing more and appends no
3189 # marker of its own: its output is exactly its two passes over
3190 # ``head``. Calling it (rather than repeating its passes) keeps any
3191 # change to it in force on the log-record path.
3192 head, marker = _truncate_for_log_redaction(value)
3193 return _redact_additional_shapes(_redact_log_text(head)) + marker
3196def redact_for_log(value: str) -> str:
3197 """Run the log patcher's credential redaction on *value* alone.
3199 The same two passes and the same ``_LOG_REDACTION_MAX_CHARS`` bound the
3200 loguru patcher applies to every record (:func:`_redact_log_text`), for a
3201 caller that is about to shorten *value* before it reaches a log line.
3202 Shortening first can hide a credential from the patcher: a cut that drops
3203 ``?key=`` or the ``@`` after URL userinfo, but keeps part of the secret,
3204 leaves text no credential pattern matches. Redacting first means any cut
3205 afterwards only removes text the patcher already judged safe or already
3206 replaced. Cost is bounded the same way as the patcher's.
3207 """
3208 return _redact_log_text(value)
3211# How much of one value :func:`redact_and_bound_for_log` scans: at least
3212# this many characters, or four times the output cap if that is more, and
3213# never more than the patcher's own ``_LOG_REDACTION_MAX_CHARS``. The
3214# result keeps at most ``max_length`` characters, so scanning the full
3215# 32 KiB made one call cost ~50-100 ms on adversarial input (a run of
3216# spaces, or of ``https://a:b@ ``) for a 256-character result; 1,024
3217# characters cost a few milliseconds.
3218_LOG_BOUNDED_REDACTION_MIN_CHARS = 1_024
3221def bounded_redaction_chars(max_length: int) -> int:
3222 """How many characters of a value to redact for a *max_length* result."""
3223 return min(
3224 _LOG_REDACTION_MAX_CHARS,
3225 max(_LOG_BOUNDED_REDACTION_MIN_CHARS, 4 * max_length),
3226 )
3229def redact_head_for_log(value: str, max_chars: int) -> str:
3230 """:func:`redact_for_log` on at most *max_chars* characters of *value*.
3232 For a caller that shows only a short prefix of the result. The value
3233 is cut first by the patcher's own rule
3234 (:func:`_truncate_for_log_redaction` with *max_chars* as the bound:
3235 only at a whitespace boundary, dropping the whole run the bound falls
3236 in), then the kept head is redacted, and the rest is replaced by the
3237 omitted-length marker. The patcher's own argument applies: a
3238 credential token holds no boundary character, so the head never keeps
3239 part of a token whose label or ``@`` was cut off, and text past the
3240 cut is never shown. The cost is bounded by *max_chars*.
3241 """
3242 if not value:
3243 return value
3244 head, marker = _truncate_for_log_redaction(value, max_chars)
3245 return redact_for_log(head) + marker
3248def redact_and_bound_for_log(value: str, max_length: int) -> str:
3249 """:func:`redact_for_log`, then :func:`sanitize_for_log` (strip + cap).
3251 Use instead of plain ``sanitize_for_log`` for a value that a call site
3252 previously logged in full, where the patcher saw (and redacted) the whole
3253 value: truncating first would let it see less than it did before.
3255 Only the first :func:`bounded_redaction_chars` characters (at least
3256 1,024) are redacted, cut by the patcher's whitespace-boundary rule (see
3257 :func:`redact_head_for_log`); the result keeps at most *max_length* of
3258 them. A value whose first whitespace-free run reaches past that bound
3259 is shown as the omitted-length marker only.
3260 """
3261 return sanitize_for_log(
3262 redact_head_for_log(value, bounded_redaction_chars(max_length)),
3263 max_length,
3264 )
3267def _is_sensitive_log_key(key: Any) -> bool:
3268 """True when *key* names a secret under settings redaction's rules.
3270 Delegates to ``DataSanitizer.is_sensitive_setting`` with
3271 :data:`LOG_SENSITIVE_KEYS`: the last dotted segment matches a name exactly
3272 or as an underscore-delimited suffix (``openai_api_key``,
3273 ``db_password``), after invisible characters are removed from it. Message
3274 text also recognizes prefixed and dotted assignments, and the additional
3275 shape pass recognizes quoted credential keys. A ``str`` subclass key is
3276 checked by its plain characters: its own ``rsplit``, ``lower`` or
3277 ``__eq__`` could otherwise hide a name. A key longer than
3278 ``_LOG_REDACTION_MAX_CHARS`` is treated as sensitive without scanning it,
3279 keeping the cost per key bounded and masking its value.
3280 """
3281 if not isinstance(key, str):
3282 return False
3283 if len(key) > _LOG_REDACTION_MAX_CHARS:
3284 return True
3285 return DataSanitizer.is_sensitive_setting(
3286 _exact_str(key), sensitive_keys=LOG_SENSITIVE_KEYS
3287 )
3290def _is_structural_extra_key(key: Any) -> bool:
3291 """True when *key* is one of the routing keys, as an exact ``str``.
3293 A ``str`` subclass can hash and compare equal to ``"username"`` while
3294 holding (and rendering) other text, so only an exact ``str`` passes
3295 unredacted as a routing key.
3296 """
3297 return type(key) is str and key in _LOG_STRUCTURAL_EXTRA_KEYS
3300def _is_empty_log_value(value: Any) -> bool:
3301 """``None`` or an empty str/list/dict (left readable, like DataSanitizer).
3303 Type-checked rather than ``value in (None, "", [], {})`` so an object with
3304 a raising ``__eq__`` cannot abort the redaction of the whole record.
3305 Exact types only: a subclass can override ``__len__`` (or iterate items
3306 its storage does not hold) to look empty under a sensitive key, so a
3307 subclass is never "empty" and is masked like any other value.
3308 """
3309 if value is None: 3309 ↛ 3310line 3309 didn't jump to line 3310 because the condition on line 3309 was never true
3310 return True
3311 return _type_is_one_of(type(value), (str, list, dict)) and len(value) == 0
3314# Nesting bound for ``extra`` values: a container at this depth or deeper
3315# (a value bound directly in ``extra`` is depth 1) becomes a placeholder.
3316_LOG_EXTRA_MAX_DEPTH = 32
3319def _scrub_log_key(key: Any) -> Any:
3320 """Scrub a dict key in ``extra`` the way a string value is scrubbed.
3322 A ``str`` key is redacted (:func:`_redact_log_text`). ``None``, ``bool``,
3323 ``int`` and ``float`` keys cannot carry a credential and pass through. A
3324 key whose text is fixed by its type (``bytes``, or a tuple or frozenset
3325 of such values, see :func:`_is_stable_text_value`) is kept as is unless
3326 its ``str()`` redacts differently, in which case the redacted text
3327 replaces it. Any other key (an object with its own ``__str__`` or
3328 ``__repr__``, a ``str`` subclass) is always replaced by its checked,
3329 redacted text: the sinks render the key again, and an object may render
3330 a secret the second time, or (when its ``str()`` raised here) render
3331 text that was never checked at all.
3332 """
3333 if isinstance(key, str):
3334 return _redact_log_text(key)
3335 if key is None or _type_is_one_of(type(key), (bool, int, float)): 3335 ↛ 3336line 3335 didn't jump to line 3336 because the condition on line 3335 was never true
3336 return key
3337 try:
3338 text = _exact_str(str(key))
3339 printable = True
3340 except Exception:
3341 text = _unprintable_placeholder(key)
3342 printable = False
3343 redacted = _redact_log_text(text)
3344 if printable and redacted == text and _is_stable_text_value(key): 3344 ↛ 3345line 3344 didn't jump to line 3345 because the condition on line 3344 was never true
3345 return key
3346 return redacted
3349def _unique_log_key(out: dict, key: Any, reserved: Any = ()) -> Any:
3350 """*key*, or ``key#2``, ``key#3``, ... when it is already taken.
3352 Redaction can map two different keys to the same text (two URLs with
3353 different passwords, or ``"a\\nb"`` and ``"ab"``); the second must not
3354 overwrite the first. *reserved* holds keys that may not be produced at
3355 all (the routing keys of the top-level ``extra``).
3357 Keys are taken in the input's iteration order, so the first key to
3358 produce a given text keeps it unsuffixed and later ones are suffixed; a
3359 reserved key is never produced, even for the first key that redacts to
3360 it (the caller keeps a routing key itself under its own name, whether
3361 it comes before or after a key that redacts to it).
3362 """
3363 if key not in out and key not in reserved:
3364 return key
3365 base = key if isinstance(key, str) else _safe_log_str(key)
3366 n = 2
3367 while f"{base}#{n}" in out or f"{base}#{n}" in reserved: 3367 ↛ 3368line 3367 didn't jump to line 3368 because the condition on line 3367 was never true
3368 n += 1
3369 return f"{base}#{n}"
3372def _scrub_log_bytes(value: Any) -> Any:
3373 """Scrub a ``bytes``/``bytearray`` ``extra`` leaf.
3375 Only the first ``4 * _LOG_REDACTION_MAX_CHARS + 4`` bytes are decoded
3376 (UTF-8, undecodable bytes become U+FFFD, which cannot join a credential
3377 name the way a ``\\xNN`` escape's hex digits would), so the cost does not
3378 grow with the value. A value that decodes to at most
3379 ``_LOG_REDACTION_MAX_CHARS`` characters is returned unchanged (a
3380 ``bytearray`` or ``bytes`` subclass as a plain copy of itself) unless
3381 :func:`_redact_log_text` redacts something in it, in which case the
3382 redacted text replaces it. A longer value is always replaced by its
3383 redacted, truncated text, ending in a marker that gives its total size
3384 in bytes.
3385 """
3386 if type(value) is not bytes: 3386 ↛ 3390line 3386 didn't jump to line 3390 because the condition on line 3386 was never true
3387 # A ``bytearray`` (or a ``bytes`` subclass) can change or render
3388 # differently after the check; check and return a plain copy (read
3389 # through the buffer, so a ``__bytes__`` override is not called).
3390 value = (
3391 bytearray(memoryview(value))
3392 if isinstance(value, bytearray)
3393 else bytes(memoryview(value))
3394 )
3395 limit = 4 * _LOG_REDACTION_MAX_CHARS + 4
3396 text = bytes(value[:limit]).decode("utf-8", "replace")
3397 head, marker = _truncate_for_log_redaction(text)
3398 if not marker:
3399 # Compared with the control-stripped text rather than ``text``:
3400 # ``_redact_log_text`` always strips control characters, so only a
3401 # difference beyond that strip is a redaction (``api_<U+001F>key=``
3402 # included), while benign bytes holding a newline stay bytes.
3403 redacted = _redact_log_text(text)
3404 if redacted == strip_control_chars(text):
3405 return value
3406 return redacted
3407 return (
3408 _redact_log_text(head)
3409 + f" [... {len(value)} bytes in total, the rest omitted from log output]"
3410 )
3413def _scrub_log_leaf(value: Any, _depth: int = 0, _active: Any = None) -> Any:
3414 """Recursively scrub credential shapes from an ``extra`` value.
3416 Redacts by key *name* (a value under a sensitive key is masked whole)
3417 AND by value *shape* (a credential embedded in a string leaf under any
3418 key is scrubbed) — the two halves that a name-only or shape-only check
3419 would each miss. Dict keys are scrubbed too (:func:`_scrub_log_key`);
3420 sets and tuples come back as lists; a ``bytes``/``bytearray`` leaf is
3421 handled by :func:`_scrub_log_bytes`. Other non-container leaves pass
3422 through.
3424 A container that contains itself (directly or further down) is rendered
3425 as ``"<cycle>"`` and a container at depth :data:`_LOG_EXTRA_MAX_DEPTH`
3426 or deeper as ``"<nested too deep>"``, so neither raises
3427 ``RecursionError`` and trips :func:`redact_log_record`'s fail-closed
3428 fallback for the whole record.
3429 """
3430 if isinstance(value, str):
3431 return _redact_log_record_text(value)
3432 if isinstance(value, (bytes, bytearray)):
3433 return _scrub_log_bytes(value)
3434 if not isinstance(value, (dict, list, tuple, set, frozenset)):
3435 return value
3436 if _depth >= _LOG_EXTRA_MAX_DEPTH:
3437 return "<nested too deep>"
3438 if _active is None: 3438 ↛ 3439line 3438 didn't jump to line 3439 because the condition on line 3438 was never true
3439 _active = set()
3440 marker = id(value)
3441 if marker in _active:
3442 return "<cycle>"
3443 _active.add(marker)
3444 try:
3445 if isinstance(value, dict):
3446 out: dict = {}
3447 for k, v in value.items():
3448 out[_unique_log_key(out, _scrub_log_key(k))] = (
3449 REDACTION_TEXT
3450 if _is_sensitive_log_key(k) and not _is_empty_log_value(v)
3451 else _scrub_log_leaf(v, _depth + 1, _active)
3452 )
3453 return out
3454 return [_scrub_log_leaf(v, _depth + 1, _active) for v in value]
3455 finally:
3456 _active.discard(marker)
3459def redact_log_extra(extra: Any) -> Any:
3460 """Redact secrets from a loguru record's structured ``extra`` mapping.
3462 * Keys the sinks depend on for routing (``research_id``, ``username``,
3463 ``policy_audit``) pass through untouched -- as exact ``str`` keys only
3464 (:func:`_is_structural_extra_key`); a ``str`` subclass key is checked
3465 by its plain characters like any other key.
3466 * A value under a sensitive key name (:data:`LOG_SENSITIVE_KEYS`, exact
3467 or as an ``_``-delimited suffix such as ``openai_api_key``) is masked
3468 whole (empty values are left readable, mirroring ``DataSanitizer``).
3469 * Other keys are scrubbed for credential shapes like string values; a
3470 key whose scrubbed form is a routing key or an earlier key gets a
3471 ``#2``/``#3`` suffix instead of overwriting it.
3472 * Every other value is recursively scrubbed for credential *shapes*, so
3473 a token/URL/query bound under an innocuous key (the family of bug the
3474 name-based checks miss) is still caught. String values get the same
3475 control-character strip as the message, between two redaction passes,
3476 and then the same ``_ADDITIONAL_CREDENTIAL_PATTERNS``
3477 (:func:`_redact_log_record_text`).
3479 Returns a new dict; the input is not mutated. A non-dict ``extra`` is
3480 returned unchanged.
3481 """
3482 if not isinstance(extra, dict): 3482 ↛ 3483line 3482 didn't jump to line 3483 because the condition on line 3482 was never true
3483 return extra
3484 out: dict = {}
3485 # ``extra`` itself counts as an enclosing container, so a value that
3486 # refers back to the mapping is reported as a cycle at the first level.
3487 active = {id(extra)}
3488 for key, value in extra.items():
3489 if _is_structural_extra_key(key):
3490 out[key] = value
3491 continue
3492 safe_key = _unique_log_key(
3493 out, _scrub_log_key(key), _LOG_STRUCTURAL_EXTRA_KEYS
3494 )
3495 if _is_sensitive_log_key(key) and not _is_empty_log_value(value):
3496 out[safe_key] = REDACTION_TEXT
3497 else:
3498 out[safe_key] = _scrub_log_leaf(value, 1, active)
3499 return out
3502def _unprintable_placeholder(obj: Any) -> str:
3503 """Stand-in text for an object whose ``str()`` raised or was not a str."""
3504 return f"<unprintable {_type_name(type(obj))}>"
3507def _safe_log_str(obj: Any) -> str:
3508 """``str(obj)``, or a placeholder when that raises.
3510 ``str()`` raises ``TypeError`` itself when ``__str__`` returns a
3511 non-str, so one guard covers both a raising and a mistyped ``__str__``.
3512 Only the one node's text is replaced; the rest of the exception graph
3513 and its traceback are still rendered. A ``__str__`` may return a ``str``
3514 subclass, which can render differently later (its own ``__str__``,
3515 ``__format__`` or ``__repr__``); the result is always an exact ``str``.
3516 """
3517 try:
3518 return _exact_str(str(obj))
3519 except Exception:
3520 return _unprintable_placeholder(obj)
3523def _exact_str(text: str) -> str:
3524 """*text* as an exact ``str`` (a copy when it is a ``str`` subclass)."""
3525 return text if type(text) is str else str.__str__(text)
3528def _plain_text(value: Any) -> Any:
3529 """*value* as an exact ``str`` when it is a ``str``; anything else as is.
3531 Called before a ``not value`` shortcut: a ``str`` subclass can override
3532 ``__bool__`` or ``__len__`` to look empty while holding text, and would
3533 then be returned unredacted. The copy's truth value and length are its
3534 real characters'.
3535 """
3536 return _exact_str(value) if isinstance(value, str) else value
3539# Line breaks and tabs that exception text keeps: ``\t``, ``\n`` and ``\r``
3540# are never drawn as nothing in a sink -- terminal, log file, and the DB log
3541# shown in the browser -- (a tab, a line break, or for ``\r`` a line break
3542# or, in a terminal, a return to the start of the line), so a credential
3543# name they split is not shown as one unbroken name, and a multi-line
3544# message keeps its lines.
3545_EXCEPTION_KEPT_BREAKS = "\t\n\r"
3547# Every other character ``strip_control_chars`` removes: zero-width and bidi
3548# format characters, and all other C0/C1 controls. That deliberately
3549# includes the ones ``str.isspace()`` calls whitespace -- ``\x1c``-``\x1f``,
3550# U+0085, ``\v``, ``\f``, U+2028 and U+2029. Terminals draw ``\x1c``-``\x1f``
3551# as nothing, browsers draw U+0085 and ``\v`` as nothing, and ``\f``,
3552# U+2028 and U+2029 are a line break in some sinks and nothing in others.
3553# Only a character that is a visible separator in every sink may stay, since
3554# any of these can hide a credential name (``api_<U+200B>key=``,
3555# ``tok<U+001F>en=``) or split URL userinfo exactly like a zero-width space.
3556# Derived from ``_UNSAFE_CHAR_RE`` so the two cannot drift apart.
3557_EXCEPTION_INVISIBLE_RE = re.compile(
3558 "["
3559 + "".join(
3560 re.escape(char)
3561 for char in map(
3562 chr,
3563 [
3564 *range(0xA0),
3565 0x61C,
3566 *range(0x2000, 0x2070),
3567 0xFEFF,
3568 ],
3569 )
3570 if _UNSAFE_CHAR_RE.match(char) and char not in _EXCEPTION_KEPT_BREAKS
3571 )
3572 + "]"
3573)
3576def _redact_exception_text(value: str) -> str:
3577 """Redact one rendered exception field (value, group message or note).
3579 The first pass sees the text as loguru will render it. The second pass
3580 runs on that output with every character :func:`strip_control_chars`
3581 removes, except ``\t``, ``\n`` and ``\r``, removed
3582 (:data:`_EXCEPTION_INVISIBLE_RE`), catching a credential name or URL
3583 those characters split; its result is used only when it redacts
3584 something more. Tabs and line breaks are kept in both passes, so a
3585 multi-line exception message keeps its lines, and a value the first
3586 pass redacted at a line end is not glued onto the next line's text.
3587 When removing those characters leaves exactly the original text (none
3588 in it, and the first pass changed nothing), the second pass would scan
3589 that same text again and return the first pass's result, so it is
3590 skipped: clean text, the common case, costs one pass. The additional
3591 credential shapes run on the selected redacted text before the omission
3592 marker is appended.
3593 """
3594 value = _plain_text(value)
3595 if not value:
3596 return _exact_str(value)
3597 head, marker = _truncate_for_log_redaction(value)
3598 first = _redact_bounded_log_text(head)
3599 visible = _EXCEPTION_INVISIBLE_RE.sub("", first)
3600 if visible == head:
3601 # ``_redact_bounded_log_text(visible)`` is ``first`` itself, which
3602 # the line below would return either way.
3603 result = first
3604 else:
3605 second = _redact_bounded_log_text(visible)
3606 result = first if second == visible else second
3607 return _exact_str(_redact_additional_shapes(result) + marker)
3610def _rendered_exception_notes(node: Any) -> tuple[list, bool]:
3611 """The notes the traceback module renders for *node*, and an exactness flag.
3613 Mirrors ``TracebackException.format_exception_only`` on Python 3.12 to
3614 3.14 (which loguru calls): a ``__notes__`` that is a
3615 ``collections.abc.Sequence`` other than ``str``/``bytes`` is iterated
3616 and each item rendered with ``str()``; any other value except ``None``
3617 (a ``str``, a ``set``, an arbitrary object) is rendered as one note,
3618 its ``repr()``. The flag is true only for a ``list`` or ``tuple`` of
3619 exact ``str`` objects, which renders as exactly the returned texts as
3620 long as reading ``__notes__`` again returns the same object; a
3621 ``__notes__`` property or ``__getattr__`` on the class is caught by
3622 :func:`_exception_text_is_stable` instead.
3623 Anything else (a ``UserList``, a note object, a non-sequence) is
3624 evaluated again when the traceback is rendered and may produce
3625 different text from what was checked, so the caller rebuilds the node
3626 with the checked texts. Raises when getting or iterating ``__notes__``
3627 raises.
3628 """
3629 notes = getattr(node, "__notes__", None)
3630 if notes is None:
3631 return [], True
3632 if isinstance(notes, collections.abc.Sequence) and not isinstance(
3633 notes, (str, bytes)
3634 ):
3635 items = list(notes)
3636 exact = _type_is_one_of(type(notes), (list, tuple)) and all(
3637 type(item) is str for item in items
3638 )
3639 return [_safe_log_str(item) for item in items], exact
3640 try:
3641 text = repr(notes)
3642 except Exception:
3643 text = None
3644 if not isinstance(text, str): 3644 ↛ 3645line 3644 didn't jump to line 3645 because the condition on line 3644 was never true
3645 text = _unprintable_placeholder(notes)
3646 return [text], False
3649# Nodes visited per record when rebuilding an exception graph (causes,
3650# contexts and group members together); the rest become placeholders.
3651_LOG_EXCEPTION_NODE_BUDGET = 200
3653# Values whose ``str()`` and ``repr()`` are fixed by their type: an
3654# exception holding only these (in ``args`` or in a field its builtin base
3655# renders) renders the same text every time it is rendered.
3656_STABLE_TEXT_TYPES = (str, int, float, complex, bool, type(None), bytes)
3658# Attributes the sinks read or call on an exception they render: its text,
3659# chained exceptions and notes, and a group's message and members (the
3660# traceback module), its truth value (loguru's ``if exc_value:``, which
3661# calls ``__bool__`` or ``__len__``), ``__class__`` (the ``isinstance``
3662# group check falls back to it), a chained exception's ``__traceback__``,
3663# and the pickling hooks (an ``enqueue=True`` sink pickles the record,
3664# exception included, after rendering it and before the next sink renders
3665# it; ``BaseException.__reduce__`` calls neither ``__getstate__`` nor
3666# ``__getnewargs__``/``__getnewargs_ex__`` today, but a kept class defines
3667# no pickling hook at all). A non-builtin class anywhere in the exception
3668# type's MRO (a mixin listed after a builtin base included) that defines
3669# one of these (a
3670# ``__str__`` override, a ``__notes__`` or ``__cause__`` property, a
3671# ``__getattr__`` that supplies ``__notes__``, a ``__len__`` or
3672# ``__reduce_ex__`` that rewrites ``args``) can render, or make a later
3673# sink render, different text from what the redactor checked.
3674_RENDER_ATTRIBUTES = frozenset(
3675 {
3676 "__str__",
3677 "__getattribute__",
3678 "__getattr__",
3679 "args",
3680 "__notes__",
3681 "__cause__",
3682 "__context__",
3683 "__suppress_context__",
3684 "message",
3685 "exceptions",
3686 "__bool__",
3687 "__len__",
3688 "__class__",
3689 "__traceback__",
3690 "__reduce__",
3691 "__reduce_ex__",
3692 "__getstate__",
3693 "__getnewargs__",
3694 "__getnewargs_ex__",
3695 }
3696)
3698# Attributes read on the exception's *type*: the traceback module renders
3699# ``__qualname__`` and ``__module__``, pickling looks the class up by them,
3700# and the stability check itself reads ``__mro__`` and ``__dict__``. A
3701# non-builtin metaclass whose class dict holds anything under one of these
3702# names (a property, a function, a callable object that is not a
3703# descriptor) can run its code or make the attribute read something other
3704# than the type's own value -- an entry under ``__name__`` or
3705# ``__qualname__``, even a plain string, shadows the type's own. The one
3706# exception is a plain ``str`` under ``__module__``, which every class
3707# statement puts in a metaclass's dict: it is inert, because the class's
3708# own ``__module__`` entry is found first.
3709_METACLASS_RENDER_ATTRIBUTES = frozenset(
3710 {
3711 "__getattribute__",
3712 "__getattr__",
3713 "__name__",
3714 "__qualname__",
3715 "__module__",
3716 "__mro__",
3717 "__dict__",
3718 }
3719)
3721# ``type``'s own descriptors for the attributes above: reading through them
3722# returns the type's real MRO (the order attribute lookup uses), class dict
3723# and names, without running a metaclass's code.
3724_TYPE_MRO = vars(type)["__mro__"]
3725_TYPE_DICT = vars(type)["__dict__"]
3726_TYPE_NAME = vars(type)["__name__"]
3727_TYPE_QUALNAME = vars(type)["__qualname__"]
3728_TYPE_MODULE = vars(type)["__module__"]
3730# Fields a builtin exception renders besides ``args`` (in ``str()`` or in
3731# the traceback module's extra lines), by the builtin base that renders
3732# them. An exception that is an instance of the base must hold only
3733# :data:`_STABLE_TEXT_TYPES` there, and no non-builtin class in the
3734# exception type's MRO may redefine them. ``SyntaxError._metadata``
3735# (Python 3.13+) holds the source the traceback module's keyword-typo
3736# suggestion reads its replacement line from. ``AttributeError.obj`` is not
3737# listed: it feeds only the "Did you mean" suggestion, and an exception for
3738# which the traceback module would compute one is never kept (see
3739# :func:`_exception_text_is_stable`).
3740_RENDERED_FIELDS = (
3741 (OSError, ("errno", "strerror", "filename", "filename2")),
3742 (
3743 SyntaxError,
3744 (
3745 "msg",
3746 "filename",
3747 "lineno",
3748 "offset",
3749 "text",
3750 "end_lineno",
3751 "end_offset",
3752 "print_file_and_line",
3753 "_metadata",
3754 ),
3755 ),
3756 (UnicodeError, ("encoding", "object", "start", "end", "reason")),
3757 (ImportError, ("msg", "name", "path", "name_from")),
3758 (NameError, ("name",)),
3759 (AttributeError, ("name",)),
3760)
3763# ``BaseException``'s ``__dict__`` descriptor: the instance dict, read
3764# without going through the instance's (possibly overridden) attribute
3765# lookup.
3766_BASE_EXCEPTION_DICT = vars(BaseException)["__dict__"]
3769# Answers computed during the current decision phase of
3770# :func:`redact_log_exception` (``None`` outside one), by the check and the
3771# identity of what it was asked about (:func:`_once_per_phase`).
3772_DECISION_MEMO: contextvars.ContextVar = contextvars.ContextVar(
3773 "ldr_log_decision_memo", default=None
3774)
3777def _once_per_phase(check: Any) -> Any:
3778 """*check*, answered once per decision phase for the same subjects.
3780 The checks wrapped with this (whether pickling a class runs code, a
3781 class's rendering hooks and names, the reducers registered for a type,
3782 ``sys.modules``'s keys) depend only on the class or object they are
3783 asked about and on global tables: class and metaclass dicts, MROs, the
3784 picklers' dispatch tables, ``sys.modules`` and module namespaces. A
3785 decision phase (:func:`_decision_phase`) runs no application code for
3786 exception classes that do not hook the machinery those checks read
3787 through, so nothing changes any of these between two nodes' checks
3788 (another thread could, which the threat model in the module docstring
3789 does not cover either way), and the first answer for a subject holds for
3790 the rest of the phase: a group of many members of one class pays for
3791 its class's checks once, not per member. The subjects are compared by
3792 identity (``id()``, so no ``__hash__`` or ``__eq__`` of theirs runs)
3793 and held by the memo, so an id cannot be reused while it lives. Outside
3794 a phase, every call computes its answer. A check that raises is not
3795 remembered.
3796 """
3798 @functools.wraps(check)
3799 def memoized(*subjects: Any) -> Any:
3800 memo = _DECISION_MEMO.get()
3801 if memo is None:
3802 return check(*subjects)
3803 key = (check, *map(id, subjects))
3804 entry = memo.get(key)
3805 if entry is None:
3806 entry = memo[key] = (subjects, check(*subjects))
3807 return entry[1]
3809 return memoized
3812@contextlib.contextmanager
3813def _decision_phase():
3814 """Scope of one record's decisions, and of :func:`_once_per_phase`'s memo.
3816 Entered by :func:`redact_log_exception` only after every step that can
3817 run application code has run (the read phase,
3818 :func:`_plan_redacted_exception`), so an answer remembered in it cannot
3819 go stale through code the redaction itself runs.
3820 """
3821 token = _DECISION_MEMO.set({})
3822 try:
3823 yield
3824 finally:
3825 _DECISION_MEMO.reset(token)
3828def _is_builtin_class(kind: Any) -> bool:
3829 """True when *kind* is the builtin class of that name (not a lookalike).
3831 The name is read through ``type``'s own descriptor, so a metaclass's
3832 ``__name__`` is never called.
3833 """
3834 try:
3835 name = _TYPE_NAME.__get__(kind)
3836 return type(name) is str and getattr(builtins, name, None) is kind
3837 except Exception:
3838 return False
3841def _type_name(kind: Any, default: str = "object") -> str:
3842 """*kind*'s own ``__name__``, redacted; *default* if it is not a type.
3844 Read through ``type``'s descriptor (a metaclass's ``__name__`` property
3845 is never called, and cannot return one name now and another later) and
3846 redacted like exception text, since a class can be created under any
3847 name (``type("token=...", (Exception,), {})``). Used wherever the
3848 redactor writes a type's name into text a sink renders.
3849 """
3850 try:
3851 name = _exact_str(_TYPE_NAME.__get__(kind))
3852 return _redact_name_text(name)
3853 except Exception:
3854 return default
3857# Longest name :func:`_redact_name_text` remembers; a longer one (a class
3858# can be created under any name) is redacted on every call.
3859_NAME_CACHE_MAX_LEN = 256
3862def _redact_name_text(name: str) -> str:
3863 """:func:`_redact_exception_text` of the exact ``str`` *name*, remembered.
3865 Class, module and qualified names are redacted on every record that
3866 renders or keeps an exception, and the same few recur. Redaction is a
3867 function of the text alone (fixed patterns, no state), so a remembered
3868 result cannot go stale; names longer than :data:`_NAME_CACHE_MAX_LEN`
3869 are not remembered, which bounds the memory the cache holds.
3870 """
3871 if len(name) > _NAME_CACHE_MAX_LEN: 3871 ↛ 3872line 3871 didn't jump to line 3872 because the condition on line 3871 was never true
3872 return _redact_exception_text(name)
3873 return _redact_short_name(name)
3876@functools.lru_cache(maxsize=1024)
3877def _redact_short_name(name: str) -> str:
3878 """:func:`_redact_exception_text` of *name* (:func:`_redact_name_text`)."""
3879 return _redact_exception_text(name)
3882def type_name_for_log(kind: Any, default: str = "Exception") -> str:
3883 """Public form of :func:`_type_name` for sinks that print a type name.
3885 The DB sink's row prefix (``utilities.log_utils._exception_context``)
3886 names ``record["exception"].type``, which the patcher leaves as the
3887 original class unless :func:`_record_type_is_safe` rejects it; this
3888 reads and redacts that name the way the patcher's own rebuilt text does,
3889 so the two agree.
3890 """
3891 return _type_name(kind, default)
3894def _rendered_block(header: str, notes: list) -> str:
3895 """The text a sink renders for one exception: its header line, then notes.
3897 ``TracebackException.format_exception_only`` writes the header line
3898 (``TypeName: text``, see :func:`_rendered_header_tails`) and then each
3899 note on lines of its own, so a credential label at the end of one and
3900 its value at the start of the next are adjacent text.
3901 """
3902 return "\n".join([header, *notes])
3905def _syntax_error_header_text(node: Any, location: bool = True) -> str:
3906 """The text a ``SyntaxError``'s header line shows after ``TypeName: ``.
3908 ``TracebackException._format_syntax_error`` prints ``msg`` (or
3909 ``<no detail available>`` when it is empty), followed by `` (filename)``
3910 only when ``lineno`` is ``None`` and ``filename`` is not; the file and
3911 line number otherwise go on the ``File ...`` line above it. That is not
3912 ``str(node)``, which adds `` (basename, line N)``: a suffix the sinks do
3913 not print between ``msg`` and the notes on the following lines. On
3914 Python 3.13+ the keyword-typo suggestion can replace an ``invalid
3915 syntax`` message, or one mentioning a forgotten comma, with the fixed
3916 text ``invalid syntax. Did you mean 'kw'?`` (``kw`` a Python keyword).
3917 With *location* false, the `` (filename)`` suffix is left out.
3918 """
3919 msg = node.msg
3920 if msg is None or (_is_stable_text_value(msg) and not msg):
3921 text = "<no detail available>"
3922 else:
3923 text = _safe_log_str(msg)
3924 if location and node.lineno is None and node.filename is not None:
3925 text += f" ({_safe_log_str(node.filename)})"
3926 return text
3929def _rendered_header_tails(node: Any, text: str) -> tuple:
3930 """What the traceback module prints after the type name for *node*.
3932 The header line is ``TypeName: text`` (``format_exception_only``), but
3933 only ``TypeName`` when *text* (the node's ``str()``) is empty. A
3934 ``SyntaxError`` (the class or a subclass, as the traceback module tests
3935 it) shows :func:`_syntax_error_header_text` instead of *text*; its form
3936 without the `` (filename)`` suffix is returned too, so that the suffix
3937 the sinks print between ``msg`` and the notes is not what keeps a label
3938 ending ``msg`` apart from its value in a note. The first item is the
3939 line the sinks print.
3940 """
3941 if issubclass(type(node), SyntaxError):
3942 return (
3943 f": {_syntax_error_header_text(node)}",
3944 f": {_syntax_error_header_text(node, location=False)}",
3945 )
3946 return (f": {text}" if text else "",)
3949def _named_text(kind: Any, text: str) -> str:
3950 """``TypeName: text`` for a rebuilt node's text, redacted.
3952 Only ``TypeName`` when *text* is empty, as the traceback module renders
3953 an exception whose ``str()`` is empty: ``TypeName: `` would put text
3954 the original rendering did not have between the name and the notes on
3955 the following lines.
3956 """
3957 name = _type_name(kind)
3958 return _redact_exception_text(f"{name}: {text}" if text else name)
3961def _type_header_changed(
3962 kind: Any, text: str, notes: list = (), tails: tuple = ()
3963) -> bool:
3964 """True when a sink's rendered ``TypeName: text`` block redacts differently.
3966 The traceback module renders ``qualname`` and the header line's rest
3967 (*tails*, :func:`_rendered_header_tails`), with ``module.`` in front
3968 outside ``__main__`` and ``builtins``, followed by the notes, one per
3969 line (:func:`_rendered_block`); the DB row renders ``name: text``. Each
3970 header form (:func:`_rendered_type_prefixes`) is checked with the notes
3971 after it.
3972 The name, the text and each note can be clean while the joined block is
3973 not: a credential pattern can span them (a class named
3974 ``InvalidAuthorization`` and the text ``abc123...`` render as an
3975 ``Authorization: abc123...`` header; a message ending in ``password=``
3976 and a note holding the value render as ``password=`` and the value on
3977 the next line). The names are read through ``type``'s own descriptors;
3978 a name that is not an exact ``str`` counts as a change.
3979 """
3980 prefixes = _rendered_type_prefixes(kind)
3981 if prefixes is None: 3981 ↛ 3982line 3981 didn't jump to line 3982 because the condition on line 3981 was never true
3982 return True
3983 tails = dict.fromkeys((f": {text}", *tails))
3984 for prefix in prefixes:
3985 for header_tail in tails:
3986 block = _rendered_block(prefix + header_tail, notes)
3987 if _redact_exception_text(block) != block:
3988 return True
3989 return False
3992@_once_per_phase
3993def _rendered_type_prefixes(kind: Any) -> Optional[tuple]:
3994 """The forms of *kind*'s name a sink renders before ``: text``.
3996 ``name`` (the DB row), ``qualname``, and ``module.qualname`` (the
3997 traceback module, except for a class of ``builtins`` or ``__main__``,
3998 which it renders as ``qualname`` alone), without repeats; ``None`` when
3999 a name is not an exact ``str``. The names are read through ``type``'s
4000 own descriptors, so no metaclass code runs.
4001 """
4002 name = _TYPE_NAME.__get__(kind)
4003 qualname = _TYPE_QUALNAME.__get__(kind)
4004 module = _TYPE_MODULE.__get__(kind)
4005 if not all(type(part) is str for part in (name, qualname, module)): 4005 ↛ 4006line 4005 didn't jump to line 4006 because the condition on line 4005 was never true
4006 return None
4007 prefixes = [name, qualname]
4008 if module not in ("builtins", "__main__"):
4009 prefixes.append(f"{module}.{qualname}")
4010 return tuple(dict.fromkeys(prefixes))
4013def _notes_fitting_header(header: str, notes: list) -> list:
4014 """*notes*, redacted together with *header* as the sinks render them.
4016 *header* is a rebuilt node's own clean header line (``TypeName: text``,
4017 see :func:`_rendered_header_tails`) and each note is clean on its own;
4018 a credential can still span a line break between them
4019 (:func:`_rendered_block`). When redacting the joined block
4020 changes it but leaves the header line as it is, the redacted rest is
4021 returned as one note (one note holding line breaks renders as the same
4022 lines as several notes). Otherwise a placeholder note is returned, or
4023 no notes when even that does not render cleanly after the header.
4024 """
4025 if not notes: 4025 ↛ 4026line 4025 didn't jump to line 4026 because the condition on line 4025 was never true
4026 return notes
4027 block = _rendered_block(header, notes)
4028 redacted = _redact_exception_text(block)
4029 if redacted == block:
4030 return notes
4031 start = header + "\n"
4032 if redacted.startswith(start):
4033 rest = [redacted[len(start) :]]
4034 block = _rendered_block(header, rest)
4035 if _redact_exception_text(block) == block: 4035 ↛ 4037line 4035 didn't jump to line 4037 because the condition on line 4035 was always true
4036 return rest
4037 placeholder = ["<exception notes omitted: they joined into a credential>"]
4038 block = _rendered_block(header, placeholder)
4039 if _redact_exception_text(block) == block: 4039 ↛ 4041line 4039 didn't jump to line 4041 because the condition on line 4039 was always true
4040 return placeholder
4041 return []
4044def _record_type_is_safe(kind: Any) -> bool:
4045 """True when a record may keep *kind* as its exception's ``type``.
4047 An ``enqueue=True`` sink pickles the whole record before queueing it,
4048 and loguru's ``RecordException.__reduce__`` pickles ``type`` by
4049 reference outside the guard it puts around ``value``. A class pickle
4050 cannot find by its name (one defined inside a function, one made with
4051 ``type()``) makes the put raise, and loguru's error handler then writes
4052 ``str(record)`` and the pickling traceback to ``sys.stderr``; that
4053 traceback's ``__context__`` is the exception being logged, unredacted,
4054 with its whole chain. The repr of the type in ``str(record)`` shows its
4055 module and qualified name, so these must also be clean.
4057 Trying the pickle here (:func:`_pickles`) is not enough on its own: it
4058 runs once, and the sink pickles again later. So *kind* is kept only
4059 when pickling it by reference runs no code that could answer
4060 differently the second time (:func:`_pickled_by_reference_runs_no_code`,
4061 the check every exception kept in the rendered graph passes too), and
4062 only then is the pickle tried.
4063 """
4064 try:
4065 if not _pickled_by_reference_runs_no_code(kind):
4066 return False
4067 if not _type_names_are_clean(kind): 4067 ↛ 4068line 4067 didn't jump to line 4068 because the condition on line 4067 was never true
4068 return False
4069 qualname = _TYPE_QUALNAME.__get__(kind)
4070 module = _TYPE_MODULE.__get__(kind)
4071 text = "{}.{}".format(module, qualname)
4072 if _redact_name_text(text) != text: 4072 ↛ 4073line 4072 didn't jump to line 4073 because the condition on line 4072 was never true
4073 return False
4074 except Exception:
4075 return False
4076 return _pickles(kind)
4079@_once_per_phase
4080def _type_names_are_clean(kind: Any) -> bool:
4081 """True when *kind*'s names are exact ``str`` that redaction leaves as is.
4083 The names are ``__name__``, ``__qualname__`` and ``__module__``, read
4084 through ``type``'s own descriptors, so no metaclass code runs.
4085 """
4086 for read in (_TYPE_NAME, _TYPE_QUALNAME, _TYPE_MODULE):
4087 text = read.__get__(kind)
4088 if type(text) is not str or _redact_name_text(text) != text: 4088 ↛ 4089line 4088 didn't jump to line 4089 because the condition on line 4088 was never true
4089 return False
4090 return True
4093# Type attributes pickling a class, or an instance of it, reads or calls:
4094# the class's ``__module__``, ``__qualname__`` and ``__name__`` (looked up
4095# with ``getattr``, so through the metaclass's ``__getattribute__``,
4096# ``__getattr__`` and data descriptors); for a class nested in another
4097# (``Outer.Inner``), the ``__get__`` that ``getattr(Outer, "Inner")`` calls
4098# when ``Inner``'s metaclass defines one; ``__hash__`` and ``__eq__``, with
4099# which the pickler's dispatch-table lookup of an instance's class hashes
4100# and compares it; and ``__reduce__`` (and ``__reduce_ex__``), which
4101# ``object.__reduce_ex__`` reads on the instance's class to see whether it
4102# is overridden.
4103_PICKLE_METACLASS_ATTRIBUTES = _METACLASS_RENDER_ATTRIBUTES | {
4104 "__get__",
4105 "__hash__",
4106 "__eq__",
4107 "__reduce__",
4108 "__reduce_ex__",
4109}
4112@_once_per_phase
4113def _pickled_by_reference_runs_no_code(kind: Any) -> bool:
4114 """True when pickling *kind*, or an instance of it, runs none of its code.
4116 An ``enqueue=True`` sink pickles the record's ``type`` and, through
4117 loguru's ``RecordException.__reduce__`` (``pickle.dumps(value)``), the
4118 exception and everything its ``args`` and instance dict hold (a group's
4119 members, through its ``args``), after that sink rendered the record and
4120 before the next sink renders it. Pickling an object of class *kind*
4121 looks *kind* up in the picklers' dispatch tables and saves it by module
4122 and qualified name; code run on the way (a metaclass hook, a reducer
4123 registered for *kind*, an enclosing class's metaclass, the module's
4124 ``__spec__``) could change what a later sink renders, or succeed in a
4125 trial pickle and fail in the sink's. So every exception kept in the
4126 rendered graph (:func:`_exception_text_is_stable`), and the record's
4127 ``type`` (:func:`_record_type_is_safe`), must pass this one check:
4129 * *kind* is a class whose metaclass's own metaclass is ``type`` (which
4130 hashes and dispatches the metaclass when *kind* itself is pickled);
4131 * no non-builtin metaclass of *kind* defines
4132 :data:`_PICKLE_METACLASS_ATTRIBUTES` (:func:`_metaclass_is_inert`;
4133 a ``__module__`` property, say, that names the real module on the
4134 first read and another on the next);
4135 * no reducer is registered for *kind* or its metaclass
4136 (:func:`_has_registered_reducer`); and
4137 * pickle's lookup of *kind* by its module and qualified name runs no
4138 code (:func:`_pickle_lookup_runs_no_code`): it finds *kind*, or
4139 pickle refuses a class defined in a function before looking it up
4140 (that pickle then fails, which loguru's ``__reduce__`` guards for a
4141 value, and :func:`_record_type_is_safe` rejects for the ``type``).
4143 The instance side (the exception's own ``__reduce_ex__``, ``args`` and
4144 instance dict) is checked by :func:`_exception_text_is_stable`.
4145 Everything is read through the builtin descriptors, so this check runs
4146 no code of *kind*'s either.
4147 """
4148 if not issubclass(type(kind), type): 4148 ↛ 4149line 4148 didn't jump to line 4149 because the condition on line 4148 was never true
4149 return False
4150 meta = type(kind)
4151 if type(meta) is not type: 4151 ↛ 4152line 4151 didn't jump to line 4152 because the condition on line 4151 was never true
4152 return False
4153 if not _metaclass_is_inert(kind, _PICKLE_METACLASS_ATTRIBUTES):
4154 return False
4155 if _has_registered_reducer(kind) or _has_registered_reducer(meta):
4156 return False
4157 qualname = _TYPE_QUALNAME.__get__(kind)
4158 module = _TYPE_MODULE.__get__(kind)
4159 if type(qualname) is not str or type(module) is not str: 4159 ↛ 4160line 4159 didn't jump to line 4160 because the condition on line 4159 was never true
4160 return False
4161 return _pickle_lookup_runs_no_code(kind, module, qualname)
4164# The reducer ``copyreg`` registers for ``complex``: the only stock entry
4165# for a type :func:`_is_plain_data` and :func:`_is_stable_text_value`
4166# accept (the pickler saves the others without consulting the tables).
4167_STOCK_COMPLEX_REDUCER = getattr(copyreg, "pickle_complex", None)
4169# Metaclass attributes a dict lookup runs on a key it compares.
4170_DICT_KEY_ATTRIBUTES = frozenset({"__hash__", "__eq__"})
4173@_once_per_phase
4174def _has_registered_reducer(kind: Any, stock: Any = None) -> bool:
4175 """True when a pickler may call a reducer other than *stock* for *kind*.
4177 ``pickle.dumps`` (which loguru pickles an exception with) looks the
4178 exact class of every object it saves, other than its builtin fast-path
4179 types (``str``, ``int``, ``list`` ...), up in ``copyreg.dispatch_table``,
4180 and the ``ForkingPickler`` the queue uses looks it up there and in its
4181 own ``_extra_reducers``; a reducer found there runs. A table that is not
4182 an exact ``dict``, or a key that is not a class whose metaclasses leave
4183 ``__hash__`` and ``__eq__`` alone (one that could claim to equal *kind*
4184 and run its own code while the lookup compares it), counts as a
4185 reducer too. Keys are compared by identity, so nothing here runs code.
4186 """
4187 try:
4188 # What ``_metaclass_is_inert`` answers for every key whose metaclass
4189 # is ``type`` itself, the usual case (see its fast path).
4190 plain_classes_inert = _plain_metaclass_is_builtin()
4191 tables = (
4192 copyreg.dispatch_table,
4193 ForkingPickler._copyreg_dispatch_table,
4194 ForkingPickler._extra_reducers,
4195 )
4196 for table in tables:
4197 if type(table) is not dict: 4197 ↛ 4198line 4197 didn't jump to line 4198 because the condition on line 4197 was never true
4198 return True
4199 for key, reducer in dict.items(table):
4200 if key is kind:
4201 if reducer is not stock:
4202 return True
4203 elif type(key) is type:
4204 # A plain class: ``_metaclass_is_inert``'s fast path.
4205 if not plain_classes_inert: 4205 ↛ 4206line 4205 didn't jump to line 4206 because the condition on line 4205 was never true
4206 if not _metaclass_is_inert(key, _DICT_KEY_ATTRIBUTES):
4207 return True
4208 elif not ( 4208 ↛ 4212line 4208 didn't jump to line 4212 because the condition on line 4208 was never true
4209 issubclass(type(key), type)
4210 and _metaclass_is_inert(key, _DICT_KEY_ATTRIBUTES)
4211 ):
4212 return True
4213 except Exception:
4214 return True
4215 return False
4218def _pickle_lookup_runs_no_code(kind: Any, module: str, qualname: str) -> bool:
4219 """True when pickle's lookup of *kind* by name runs no code.
4221 That is, pickle refuses the name before looking anything up (a class
4222 defined in a function, whose qualified name has a ``<locals>`` part),
4223 or it finds *kind* and runs no code on the way. Pickle saves a class as
4224 its module and qualified name: it imports the module
4225 (:func:`_module_import_runs_no_code`) and checks that following the
4226 name (``getattr`` for each part of ``qualname``) gives back the class. Each ``getattr`` runs the code of the object it is called on: a
4227 module that is an instance of a ``ModuleType`` subclass, or an
4228 enclosing class whose metaclass defines ``__getattribute__``,
4229 ``__getattr__``, ``__get__`` or an attribute of the part's name, could
4230 find *kind* when :func:`_pickles` tries it and fail when the
4231 ``enqueue=True`` sink pickles it, or change what a later sink renders.
4232 So the module must already be in ``sys.modules`` as an exact
4233 ``ModuleType``, each part must be found as the attribute lookup finds it
4234 (no entry of that name in the lookup object's type's MRO, then the
4235 module's dict or the first dict of the enclosing class's MRO holding
4236 the name), through dicts whose keys are all exact ``str`` (a key of
4237 another type could compare equal to the part by running its own code),
4238 each enclosing class's metaclasses must be inert
4239 (:func:`_metaclass_is_inert`), and the last part must be *kind* itself.
4240 Everything is read through the builtin descriptors, so nothing here runs
4241 code either.
4242 """
4243 parts = qualname.split(".")
4244 if "<locals>" in parts:
4245 # Pickle refuses a class defined in a function by its qualified
4246 # name alone, before it imports or looks anything up: no code runs
4247 # (and the pickle fails, which :func:`_pickles` reports).
4248 return True
4249 if not _module_import_runs_no_code(module):
4250 return False
4251 owner = sys.modules.get(module)
4252 if type(owner) is not types.ModuleType: 4252 ↛ 4253line 4252 didn't jump to line 4253 because the condition on line 4252 was never true
4253 return False
4254 for index, part in enumerate(parts):
4255 if index and not (
4256 issubclass(type(owner), type)
4257 and _metaclass_is_inert(owner, _PICKLE_METACLASS_ATTRIBUTES)
4258 ):
4259 return False
4260 # Any entry under ``part`` in the type of ``owner`` (the module
4261 # type, or the enclosing class's metaclass) is found before
4262 # ``owner``'s own: a data descriptor there wins.
4263 for cls in _TYPE_MRO.__get__(type(owner)):
4264 namespace = _TYPE_DICT.__get__(cls)
4265 if not _plain_names(namespace) or part in namespace: 4265 ↛ 4266line 4265 didn't jump to line 4266 because the condition on line 4265 was never true
4266 return False
4267 if index:
4268 namespaces = [
4269 _TYPE_DICT.__get__(cls) for cls in _TYPE_MRO.__get__(owner)
4270 ]
4271 else:
4272 namespaces = [_MODULE_DICT.__get__(owner)]
4273 for namespace in namespaces:
4274 if not _plain_names(namespace): 4274 ↛ 4275line 4274 didn't jump to line 4275 because the condition on line 4274 was never true
4275 return False
4276 if part in namespace:
4277 owner = namespace[part]
4278 break
4279 else:
4280 return False
4281 return owner is kind
4284# ``ModuleType``'s ``__dict__`` descriptor: a module's own namespace.
4285_MODULE_DICT = vars(types.ModuleType)["__dict__"]
4287# ``ModuleSpec``'s ``__dict__`` descriptor: a spec's instance attributes.
4288_MODULE_SPEC_DICT = vars(ModuleSpec)["__dict__"]
4290# Attributes the import system reads on a module's ``__spec__``.
4291_MODULE_SPEC_ATTRIBUTES = frozenset(
4292 {"_initializing", "__getattribute__", "__getattr__"}
4293)
4296def _module_spec_read_is_plain(kind: Any) -> bool:
4297 """True when reading ``__spec__`` on a module of class *kind* runs no code.
4299 *kind* is ``ModuleType`` or a subclass whose own classes (other than
4300 ``ModuleType`` and ``object``) define neither ``__getattribute__`` nor
4301 an entry named ``__spec__``, so the read finds the module's own
4302 ``__spec__`` entry through ``ModuleType``'s lookup.
4303 """
4304 if not issubclass(kind, types.ModuleType): 4304 ↛ 4305line 4304 didn't jump to line 4305 because the condition on line 4304 was never true
4305 return False
4306 for cls in _TYPE_MRO.__get__(kind):
4307 if cls is types.ModuleType or cls is object: 4307 ↛ 4309line 4307 didn't jump to line 4309 because the condition on line 4307 was always true
4308 continue
4309 namespace = _TYPE_DICT.__get__(cls)
4310 if not _plain_names(namespace):
4311 return False
4312 if "__getattribute__" in namespace or "__spec__" in namespace:
4313 return False
4314 return True
4317@_once_per_phase
4318def _sys_modules_names_are_plain(modules: dict) -> bool:
4319 """:func:`_plain_names` of *modules* (``sys.modules``), once per record.
4321 :func:`_module_import_runs_no_code` runs for every class kept in the
4322 rendered graph, and ``sys.modules`` holds thousands of entries in the
4323 web process, so scanning it for each one made a large exception group
4324 cost tens of milliseconds on the logging thread. Inside a decision
4325 phase it is scanned once (:func:`_once_per_phase`); it is scanned again
4326 if ``sys.modules`` is no longer the dict that was scanned.
4327 """
4328 return _plain_names(modules)
4331def _module_import_runs_no_code(module: str) -> bool:
4332 """True when pickle's import of the imported module *module* runs no code.
4334 Pickle imports the module of every class it saves by reference. For a
4335 module already in ``sys.modules`` the import system still reads the
4336 module's ``__spec__`` and that spec's ``_initializing`` flag (and its
4337 truth value), and for a dotted name it does the same for the top-level
4338 package: a ``__spec__`` object with its own ``__getattr__`` or an
4339 ``_initializing`` property, or a module without ``__spec__`` whose
4340 ``__getattr__`` then supplies one, runs code there. So each of those
4341 modules must be in ``sys.modules`` (whose keys must all be exact
4342 ``str``) as a module whose class reads ``__spec__`` from its namespace
4343 (:func:`_module_spec_read_is_plain`; a lazy-loading ``ModuleType``
4344 subclass with only a ``__getattr__`` for missing names does), and that
4345 namespace (exact ``str`` keys only) must hold ``__spec__`` as ``None``
4346 or an exact ``ModuleSpec`` whose class leaves those attributes alone
4347 and whose ``_initializing``, if set, is an exact ``bool``. The module
4348 a class is then looked up in must also be an exact ``ModuleType``
4349 (:func:`_pickle_lookup_runs_no_code`). ``sys.modules``'s keys are
4350 scanned once per record, not once per call
4351 (:func:`_sys_modules_names_are_plain`).
4352 """
4353 modules = sys.modules
4354 if type(modules) is not dict or not _sys_modules_names_are_plain(modules):
4355 return False
4356 for name in {module, module.partition(".")[0]}:
4357 owner = modules.get(name)
4358 if not _module_spec_read_is_plain(type(owner)): 4358 ↛ 4359line 4358 didn't jump to line 4359 because the condition on line 4358 was never true
4359 return False
4360 namespace = _MODULE_DICT.__get__(owner)
4361 if not _plain_names(namespace) or "__spec__" not in namespace: 4361 ↛ 4362line 4361 didn't jump to line 4362 because the condition on line 4361 was never true
4362 return False
4363 spec = namespace["__spec__"]
4364 if spec is None:
4365 continue
4366 if type(spec) is not ModuleSpec:
4367 return False
4368 for cls in _TYPE_MRO.__get__(ModuleSpec):
4369 if _MODULE_SPEC_ATTRIBUTES.intersection(_TYPE_DICT.__get__(cls)):
4370 # ``object`` defines ``__getattribute__``: builtin.
4371 if not _is_builtin_class(cls): 4371 ↛ 4372line 4371 didn't jump to line 4372 because the condition on line 4371 was never true
4372 return False
4373 state = _MODULE_SPEC_DICT.__get__(spec)
4374 # An exact ``dict``: a subclass's ``__iter__``, ``__contains__`` or
4375 # ``__getitem__`` would run here.
4376 if type(state) is not dict or not _plain_names(state):
4377 return False
4378 if ( 4378 ↛ 4382line 4378 didn't jump to line 4382 because the condition on line 4378 was never true
4379 "_initializing" in state
4380 and type(state["_initializing"]) is not bool
4381 ):
4382 return False
4383 return True
4386def _picklable_extra(extra: Any) -> Any:
4387 """*extra*, with every value that is not plain data replaced.
4389 An ``enqueue=True`` sink pickles the whole record before queueing it.
4390 One value it cannot pickle (a lock, an open file, an instance of a class
4391 defined in a function) makes the put raise, and loguru's error handler
4392 then writes ``str(record)`` and the pickling traceback to
4393 ``sys.stderr``; that traceback's ``__context__`` is the exception being
4394 handled when the log call was made (whether or not the call logged it),
4395 unredacted. Trying the pickle here would not settle it: an object's own
4396 pickling code (``__reduce_ex__``, ``__getstate__``) can succeed now and
4397 raise when the sink pickles it. So every value that is not plain data
4398 (:func:`_is_plain_data`: an exact ``str``, ``int``, ``float``,
4399 ``complex`` (while ``copyreg``'s own reducer pickles it), ``bool``,
4400 ``None``, ``bytes`` or ``bytearray``, or an exact ``list``, ``tuple``,
4401 ``dict``, ``set`` or ``frozenset`` of plain data),
4402 whose pickling runs builtin code only, is replaced, for every sink, by
4403 ``<unpicklable TypeName>``. If a key is not plain data, only the routing
4404 keys (:data:`_LOG_STRUCTURAL_EXTRA_KEYS`) are kept, and ``{}`` if their
4405 values are not plain data either.
4406 """
4407 if _is_plain_data(extra):
4408 return extra
4409 try:
4410 out = {
4411 key: value
4412 if _is_plain_data(value)
4413 else f"<unpicklable {_type_name(type(value))}>"
4414 for key, value in dict.items(extra)
4415 }
4416 except Exception:
4417 out = None
4418 if out is None or not _is_plain_data(out): 4418 ↛ 4419line 4418 didn't jump to line 4419 because the condition on line 4418 was never true
4419 out = _structural_extra_only(extra)
4420 return out if _is_plain_data(out) else {}
4423# How deep :func:`_is_plain_data` follows nested containers; a deeper one is
4424# not counted as plain data. ``redact_log_extra`` already replaces a
4425# container nested :data:`_LOG_EXTRA_MAX_DEPTH` deep, except under a
4426# routing key, which it passes through as is.
4427_PLAIN_DATA_MAX_DEPTH = _LOG_EXTRA_MAX_DEPTH + 2
4429_PLAIN_DATA_SCALARS = (
4430 str,
4431 int,
4432 float,
4433 complex,
4434 bool,
4435 type(None),
4436 bytes,
4437 bytearray,
4438)
4439_PLAIN_DATA_CONTAINERS = (list, tuple, set, frozenset)
4442def _is_plain_data(value: Any, _depth: int = 0, _seen: Any = None) -> bool:
4443 """True when pickling *value* runs builtin code only.
4445 *value* is an instance of exactly one of :data:`_PLAIN_DATA_SCALARS`, or
4446 an exact ``list``, ``tuple``, ``set``, ``frozenset`` or ``dict`` whose
4447 items (keys and values) are plain data, nested at most
4448 :data:`_PLAIN_DATA_MAX_DEPTH` deep (a ``complex`` only while
4449 :func:`_complex_pickles_as_stock`). Types are compared by identity
4450 (:func:`_type_is_one_of`), so no subclass and no metaclass ``__eq__``
4451 counts, and nothing here runs code of *value*'s own. A container that
4452 contains itself is not counted as plain data; one reached again by
4453 another path is checked once (so shared sub-containers cost linear,
4454 not exponential, time).
4455 """
4456 kind = type(value)
4457 if _type_is_one_of(kind, _PLAIN_DATA_SCALARS):
4458 return kind is not complex or _complex_pickles_as_stock()
4459 if _depth >= _PLAIN_DATA_MAX_DEPTH: 4459 ↛ 4460line 4459 didn't jump to line 4460 because the condition on line 4459 was never true
4460 return False
4461 if kind is not dict and not _type_is_one_of(kind, _PLAIN_DATA_CONTAINERS):
4462 return False
4463 if _seen is None:
4464 _seen = {}
4465 marker = id(value)
4466 if marker in _seen:
4467 # ``False`` while it is still being checked: a cycle.
4468 return _seen[marker]
4469 _seen[marker] = False
4470 if kind is dict:
4471 result = all(
4472 _is_plain_data(key, _depth + 1, _seen)
4473 and _is_plain_data(item, _depth + 1, _seen)
4474 for key, item in dict.items(value)
4475 )
4476 else:
4477 result = all(_is_plain_data(item, _depth + 1, _seen) for item in value)
4478 _seen[marker] = result
4479 return result
4482def _complex_pickles_as_stock() -> bool:
4483 """True when pickling a ``complex`` calls ``copyreg``'s own reducer only.
4485 The pickler saves ``str``, ``int``, ``float``, ``bool``, ``None``,
4486 ``bytes``, ``bytearray`` and the exact builtin containers itself, but
4487 looks ``complex`` up in its dispatch tables, where a reducer registered
4488 in place of ``copyreg``'s would run (:func:`_has_registered_reducer`).
4489 """
4490 return _STOCK_COMPLEX_REDUCER is not None and not _has_registered_reducer(
4491 complex, _STOCK_COMPLEX_REDUCER
4492 )
4495def _pickles(obj: Any) -> bool:
4496 """True when *obj* pickles the way an ``enqueue=True`` sink pickles it.
4498 loguru queues the record with ``multiprocessing.SimpleQueue.put``, which
4499 serializes with :class:`multiprocessing.reduction.ForkingPickler`. That
4500 pickler has its own reducers on top of :mod:`pickle`'s (registered, for
4501 example, for a ``ctypes`` type once ``multiprocessing.sharedctypes`` has
4502 created a value of it), so ``pickle.dumps`` succeeding does not mean
4503 the put will.
4504 """
4505 try:
4506 ForkingPickler.dumps(obj)
4507 except Exception:
4508 return False
4509 return True
4512def _is_stable_text_value(value: Any, _depth: int = 0) -> bool:
4513 """True when *value* renders the same text every time.
4515 An exact :data:`_STABLE_TEXT_TYPES` instance, or a ``tuple`` or
4516 ``frozenset`` (exact types) of such values. A subclass of ``str`` or
4517 ``int``, or any other object, may render differently on each call. A
4518 ``complex`` counts only while the reducer pickle calls for it is
4519 ``copyreg``'s own (:func:`_complex_pickles_as_stock`): it is the one
4520 such type the pickler dispatches through ``copyreg.dispatch_table``.
4521 """
4522 kind = type(value)
4523 if _type_is_one_of(kind, _STABLE_TEXT_TYPES):
4524 return kind is not complex or _complex_pickles_as_stock()
4525 if _type_is_one_of(kind, (tuple, frozenset)) and _depth < 4:
4526 return all(_is_stable_text_value(item, _depth + 1) for item in value)
4527 return False
4530def _plain_names(namespace: Any) -> bool:
4531 """True when every key of the class dict *namespace* is an exact ``str``.
4533 ``type(name) is str`` for every key, run in C (``map`` and
4534 ``operator.is_``) rather than as a generator: ``sys.modules`` is checked
4535 once per record and holds thousands of keys in the web process.
4536 """
4537 return all(map(operator.is_, map(type, namespace), itertools.repeat(str)))
4540def _plain_metaclass_is_builtin() -> bool:
4541 """True when :func:`_is_builtin_class` holds for ``type`` and ``object``.
4543 They are the whole MRO of ``type``, the metaclass of a class statement
4544 without one, so :func:`_metaclass_is_inert` holds for every class whose
4545 metaclass is exactly ``type`` (it skips builtin classes).
4546 """
4547 return (
4548 getattr(builtins, "type", None) is type
4549 and getattr(builtins, "object", None) is object
4550 )
4553def _metaclass_is_inert(kind: Any, attributes: frozenset) -> bool:
4554 """True when no non-builtin metaclass of *kind* defines *attributes*.
4556 Every metaclass in ``type(kind)``'s MRO that is not a builtin class must
4557 have a class dict whose keys are all exact ``str`` (a subclass with its
4558 own ``__hash__``/``__eq__`` could hide from the check) and that holds
4559 none of *attributes* -- except a plain ``str`` under ``__module__``,
4560 which every class statement puts in a metaclass's dict and which is
4561 inert, because the class's own ``__module__`` entry is found first. The
4562 MRO and class dicts are read through ``type``'s own descriptors, so no
4563 metaclass code runs here.
4564 """
4565 if type(kind) is type and _plain_metaclass_is_builtin():
4566 # The common case, answered without the loop: ``type``'s MRO is
4567 # ``type`` and ``object``, both builtin, so the loop skips both.
4568 return True
4569 for meta in _TYPE_MRO.__get__(type(kind)):
4570 if _is_builtin_class(meta):
4571 continue
4572 namespace = _TYPE_DICT.__get__(meta)
4573 if not _plain_names(namespace): 4573 ↛ 4574line 4573 didn't jump to line 4574 because the condition on line 4573 was never true
4574 return False
4575 for name in attributes.intersection(namespace):
4576 if name == "__module__" and type(namespace[name]) is str:
4577 continue
4578 return False
4579 return True
4582def _group_args_are_its_members(node: Any) -> bool:
4583 """True when an exception group's ``args`` are its message and members.
4585 An ``enqueue=True`` sink pickles a group through its ``args`` (the
4586 message and the container it was built from), not its ``exceptions``,
4587 after that sink rendered it and before the next sink renders it. The
4588 container is the caller's own object: a ``list`` subclass whose
4589 ``__reduce_ex__`` rewrites a member's ``args`` would change the text
4590 the later sinks render, and a list appended to after the group was
4591 built would pickle objects never checked. So ``args`` must be an exact
4592 2-tuple of the message (an exact ``str`` equal to ``message``) and an
4593 exact ``list`` or ``tuple`` holding exactly the group's members, the
4594 same objects in the same order; the members themselves are checked as
4595 nodes of their own.
4596 """
4597 args = node.args
4598 if type(args) is not tuple or len(args) != 2: 4598 ↛ 4599line 4598 didn't jump to line 4599 because the condition on line 4598 was never true
4599 return False
4600 message, members = args
4601 if type(message) is not str or message != _exact_str(node.message): 4601 ↛ 4602line 4601 didn't jump to line 4602 because the condition on line 4601 was never true
4602 return False
4603 if not _type_is_one_of(type(members), (list, tuple)):
4604 return False
4605 exceptions = node.exceptions
4606 return len(members) == len(exceptions) and all(
4607 member is exception for member, exception in zip(members, exceptions)
4608 )
4611def _exception_text_is_stable(node: Any, is_group: bool) -> bool:
4612 """True when rendering *node* again yields exactly the text checked.
4614 The redactor reads an exception's text once and keeps the original
4615 object only when that text was clean, but the sinks render the object
4616 again (loguru's ``format_exception_only``, the DB row's ``str()``) and
4617 run its code while doing so. Keeping it is safe only when no code the
4618 exception's own classes define runs then, and the values the builtin
4619 rendering reads cannot render other text. That holds when:
4621 * every non-builtin class in ``type(node).__mro__`` (including a mixin
4622 listed after a builtin base) leaves the rendering attributes alone
4623 (:data:`_RENDER_ATTRIBUTES` and the base's :data:`_RENDERED_FIELDS`);
4624 * the type is one that pickling by reference runs no code of
4625 (:func:`_pickled_by_reference_runs_no_code`, the check the record's
4626 ``type`` passes too): an ``enqueue=True`` sink pickles every kept
4627 node an exception's ``args`` reach (a group's members) after it
4628 rendered the record and before the next sink renders it, and the
4629 check is applied to every kept node, so one node can be kept or
4630 rebuilt only one way wherever it appears in the graph. Among other
4631 things, no non-builtin metaclass of the type defines the type
4632 attributes the sinks, the pickler and this check read
4633 (:data:`_PICKLE_METACLASS_ATTRIBUTES`, which include
4634 :data:`_METACLASS_RENDER_ATTRIBUTES`; the MRO, class dicts and names
4635 are read here through ``type``'s own descriptors, so a metaclass
4636 cannot hide a class or override from this check), and no class or
4637 module pickle looks it up through runs code;
4638 * the type's own ``__name__``, ``__qualname__`` and ``__module__``
4639 (which the traceback module and the DB row render) are plain ``str``
4640 that redaction leaves unchanged;
4641 * the values the builtin rendering reads (``args``, a group's
4642 ``message``, the base's fields) and the instance attributes an
4643 ``enqueue=True`` sink pickles are of :data:`_STABLE_TEXT_TYPES`
4644 (``__notes__`` aside, which the caller checks), and a group's
4645 ``args``, which that sink pickles in place of its ``exceptions``,
4646 hold nothing but its message and a plain container of exactly its
4647 members (:func:`_group_args_are_its_members`); and
4648 * the traceback module computes no "Did you mean" suggestion for it (an
4649 ``AttributeError`` or ``NameError`` with a ``str`` ``name``, an
4650 ``ImportError`` with a ``str`` ``name_from``): that runs code of
4651 other objects -- ``obj.__dir__()``, importing the named module and
4652 calling its ``__dir__``, and, for a renderer that passes the
4653 traceback (loguru does not), ``hasattr`` on the failing frame's
4654 ``self`` -- which can change the exception after its text was
4655 checked. A rebuilt node has no ``name``, so its traceback shows no
4656 suggestion.
4658 Otherwise (a ``__str__`` that returns benign text first and a secret
4659 later, a ``__notes__`` property, a ``__len__`` that rewrites ``args``,
4660 an argument object with its own ``__str__``, a class nested in one
4661 whose metaclass hooks attribute lookup) the caller rebuilds the node
4662 from the text it checked. Code that changes the exception from outside
4663 its rendering and pickling is not covered: another thread; the class's
4664 own construction code (``__init__``, ``__new__``, ``__setattr__``, a
4665 metaclass ``__call__``), which runs when an ``enqueue=True`` sink
4666 unpickles its own copy in its writer thread; a reducer registered for
4667 a builtin exception type, which also runs for the rebuilt nodes; and
4668 patches to the pickling or import machinery itself. Nor is a class or
4669 object written to defeat this check (see the module docstring's threat
4670 model): the reads here are made through builtin descriptors and exact
4671 type tests (an instance ``__dict__`` must be an exact ``dict``, a
4672 ``name`` an exact ``str``) as defence in depth, not as a guarantee.
4673 """
4674 kind = type(node)
4675 # Everything that depends on the class alone, checked once per class in
4676 # a decision phase.
4677 if not _exception_class_is_stable(kind):
4678 return False
4679 # ``type(...) is str``, not ``isinstance``: the latter falls back to a
4680 # non-``str`` value's ``__class__``, which runs its code. Any value
4681 # other than an exact ``str`` or ``None`` fails the field check below.
4682 if issubclass(kind, ImportError):
4683 if type(getattr(node, "name_from", None)) is str:
4684 return False
4685 if issubclass(kind, (NameError, AttributeError)):
4686 if type(getattr(node, "name", None)) is str:
4687 return False
4688 # Read through ``BaseException``'s own descriptor: the attribute lookup
4689 # is guarded (:func:`_exception_class_is_stable`), and this is the dict
4690 # pickling copies. It must be an exact ``dict``
4691 # (``exc.__dict__ = SomeDictSubclass()`` is accepted), read with
4692 # ``dict``'s own methods, so no ``__len__`` or ``items`` of the node's
4693 # runs here or in the pickler.
4694 state = _BASE_EXCEPTION_DICT.__get__(node)
4695 if type(state) is not dict:
4696 return False
4697 if state:
4698 for name, value in dict.items(state):
4699 if type(name) is not str: 4699 ↛ 4700line 4699 didn't jump to line 4700 because the condition on line 4699 was never true
4700 return False
4701 if name != "__notes__" and not _is_stable_text_value(value):
4702 return False
4703 if is_group:
4704 if type(node.message) is not str: 4704 ↛ 4705line 4704 didn't jump to line 4705 because the condition on line 4704 was never true
4705 return False
4706 if not _group_args_are_its_members(node):
4707 return False
4708 elif not _is_stable_text_value(node.args):
4709 return False
4710 return all(
4711 _is_stable_text_value(getattr(node, name, None))
4712 for name in _rendered_fields(kind)
4713 )
4716def _rendered_fields(kind: Any) -> list:
4717 """The :data:`_RENDERED_FIELDS` of the builtin bases *kind* derives from."""
4718 # ``issubclass`` on the type, not ``isinstance`` on the node: the latter
4719 # falls back to the node's ``__class__``, which runs its code.
4720 return [
4721 name
4722 for base, names in _RENDERED_FIELDS
4723 if issubclass(kind, base)
4724 for name in names
4725 ]
4728@_once_per_phase
4729def _exception_class_is_stable(kind: Any) -> bool:
4730 """The class-level conditions of :func:`_exception_text_is_stable`.
4732 No non-builtin class in *kind*'s MRO defines a rendering attribute,
4733 pickling *kind* by reference runs no code, and its names are clean.
4734 These depend on the class alone, so a decision phase checks them once
4735 per class (:func:`_once_per_phase`).
4736 """
4737 guarded = _RENDER_ATTRIBUTES.union(_rendered_fields(kind))
4738 # Every non-builtin class in the MRO counts, not only those before the
4739 # first builtin: in ``class E(ValueError, Mixin)`` the order is E,
4740 # ValueError, Exception, BaseException, Mixin, object, and the builtins
4741 # do not define ``__str__``, ``__notes__``, ``__getattr__`` or
4742 # ``__getattribute__`` themselves, so Python still finds Mixin's.
4743 # The MRO and class dicts are read through ``type``'s own descriptors:
4744 # ``kind.__mro__`` and ``vars(cls)`` go through the metaclass, which
4745 # could report builtin bases only, or an empty dict.
4746 # A class dict key that is not an exact ``str`` (a subclass with its
4747 # own ``__hash__``/``__eq__``) could hide from the name checks.
4748 for cls in _TYPE_MRO.__get__(kind):
4749 if _is_builtin_class(cls):
4750 continue
4751 namespace = _TYPE_DICT.__get__(cls)
4752 if not _plain_names(namespace) or guarded.intersection(namespace):
4753 return False
4754 # The sinks read the type's names and an ``enqueue=True`` sink pickles
4755 # the kept node, so its class must be one that rendering and pickling
4756 # by reference run no code of (a superset of the metaclass attributes
4757 # rendering alone reads).
4758 if not _pickled_by_reference_runs_no_code(kind):
4759 return False
4760 return _type_names_are_clean(kind)
4763# A ``SyntaxError``'s fields the interpreter fills, by the exact type it
4764# fills them with (or ``None``). Only a node whose fields all have these
4765# types is kept; anything else (a ``str`` line number, ``bytes`` source
4766# text, a ``str`` offset) is rendered differently by each Python version's
4767# traceback module, and on Python 3.12 a ``bytes`` text or non-``int``
4768# offset makes it raise, so loguru's error handler prints the context of
4769# that error -- the exception being handled -- unredacted to stderr. The
4770# ``int`` fields must also be in the accepted ranges
4771# (:func:`_syntax_error_fields_out_of_range`).
4772_SYNTAX_ERROR_FIELD_TYPES = (
4773 ("msg", str),
4774 ("filename", str),
4775 ("text", str),
4776 ("lineno", int),
4777 ("offset", int),
4778 ("end_lineno", int),
4779 ("end_offset", int),
4780)
4782# The largest line number a kept ``SyntaxError`` may carry: the interpreter
4783# fills ``lineno`` and ``end_lineno`` from a C ``int``. A larger one is not
4784# an interpreter value, and one past ``sys.get_int_max_str_digits()`` digits
4785# makes the ``File ...`` line raise ``ValueError`` when it is printed.
4786_SYNTAX_ERROR_MAX_LINENO = 2**31 - 1
4788# How far past the UTF-8 length of ``text`` (plus one) a kept
4789# ``SyntaxError``'s ``offset`` and ``end_offset`` may run, in columns. The
4790# interpreter counts columns in UTF-8 bytes of the reported line, not in
4791# characters: the tokenizer passes raw byte offsets, and the compiler and
4792# symtable stages (errors raised when a file is compiled or imported) report
4793# byte offsets too. Those stages also report the ``end_offset`` of a span
4794# that ends on a later line as a column of that later line, which ``text``
4795# does not hold and which can be longer than it. This slack covers such a
4796# later line of up to this many bytes; a span ending further right than that
4797# is not kept (the caller rebuilds the node). It also bounds the caret line
4798# Python 3.12 draws (``'^' * (end_offset - offset)``), so a value like
4799# ``10**9`` is rejected before anything is rendered
4800# (:func:`_syntax_error_fields_out_of_range`).
4801_SYNTAX_ERROR_SPAN_END_SLACK = 2**16
4803# How much longer than the text it prints verbatim (``msg``, ``filename``
4804# and ``text``, each counted three times) plus its caret line (at most its
4805# largest offset, when ``text`` is a ``str``) a kept ``SyntaxError``'s
4806# rendering may be, in characters. On Python 3.13+ the keyword-typo
4807# suggestion may replace ``msg`` and the source line with ones from at most
4808# 1024 characters of source. Anything longer is not a rendering of these
4809# fields, and the caller rebuilds the node
4810# (:func:`_syntax_error_renders_bounded`).
4811_SYNTAX_ERROR_RENDER_SLACK = 4096
4814def _syntax_error_fields_out_of_range(node: Any) -> bool:
4815 """True when a ``SyntaxError``'s ``int`` fields are not interpreter values.
4817 Called only for fields of the canonical types
4818 (:data:`_SYNTAX_ERROR_FIELD_TYPES`) and a ``text`` of at most
4819 ``_LOG_REDACTION_MAX_CHARS`` characters, so encoding it is bounded. The traceback module does
4820 arithmetic on the offsets to draw the caret line (``'^' * (end - start)``
4821 on Python 3.12, and on 3.14 too when ``text`` is empty), so an offset
4822 near ``2**63`` makes it raise ``OverflowError`` or ``MemoryError``, and a
4823 moderate one (``10**9``) makes every sink build a caret line that long.
4824 Loguru's error handler then prints the context of that error -- the
4825 exception being handled -- unredacted to stderr. The interpreter
4826 reports ``lineno`` and ``end_lineno`` between ``0`` and
4827 :data:`_SYNTAX_ERROR_MAX_LINENO`. It reports ``offset`` and
4828 ``end_offset`` as ``-1`` (unknown) or as columns counted in
4829 UTF-8 bytes, not characters, so on a line with non-ASCII text they can
4830 exceed ``len(text) + 1``; and for an error raised when a file is
4831 compiled or imported, an ``end_offset`` on a later line than
4832 ``lineno`` is a column of that later line, not of ``text``. The
4833 accepted range is therefore ``-1`` to the UTF-8 length of ``text``
4834 plus ``1`` plus :data:`_SYNTAX_ERROR_SPAN_END_SLACK`, or up to
4835 :data:`_SYNTAX_ERROR_MAX_LINENO` when ``text`` is ``None`` (the offsets
4836 are then not drawn). Anything outside these ranges makes the caller
4837 rebuild the node, which drops these fields; that includes a real error
4838 whose span ends more than :data:`_SYNTAX_ERROR_SPAN_END_SLACK` bytes
4839 into a later line.
4840 """
4841 for value in (node.lineno, node.end_lineno):
4842 if value is not None and not 0 <= value <= _SYNTAX_ERROR_MAX_LINENO:
4843 return True
4844 limit = (
4845 _SYNTAX_ERROR_MAX_LINENO
4846 if node.text is None
4847 else len(node.text.encode("utf-8", "surrogatepass"))
4848 + 1
4849 + _SYNTAX_ERROR_SPAN_END_SLACK
4850 )
4851 for value in (node.offset, node.end_offset):
4852 if value is not None and not -1 <= value <= limit:
4853 return True
4854 return False
4857def _syntax_error_renders_bounded(node: Any) -> bool:
4858 """True when the traceback module renders *node*'s lines within bounds.
4860 Runs ``traceback.format_exception_only`` -- what loguru's formatter
4861 calls -- on a detached builtin ``SyntaxError`` carrying *node*'s
4862 rendered fields (``msg``, ``filename``, ``lineno``, ``offset``,
4863 ``text``, ``end_lineno``, ``end_offset`` and ``_metadata``), not on
4864 *node* itself, so no chained exception is formatted and no frame's
4865 source is read. Called only for a node that is about to be kept, whose
4866 fields have passed :func:`_syntax_error_details_changed` (canonical
4867 types, in range, and a keyword-typo source that is an exact ``str``, so
4868 the rendering reads no file). False when that rendering raises
4869 (``OverflowError``, ``MemoryError``, ``ValueError`` or any other
4870 ``Exception`` a sink would hit, whose error handler prints the
4871 exception being handled unredacted) or is longer than
4872 :data:`_SYNTAX_ERROR_RENDER_SLACK` allows; the caller then rebuilds the
4873 node. ``KeyboardInterrupt`` and ``SystemExit`` propagate.
4874 """
4875 import traceback
4877 try:
4878 probe = SyntaxError()
4879 verbatim = 0
4880 for name, _kind in _SYNTAX_ERROR_FIELD_TYPES:
4881 value = getattr(node, name)
4882 setattr(probe, name, value)
4883 if type(value) is str:
4884 verbatim += len(value)
4885 metadata = getattr(node, "_metadata", None)
4886 if metadata is not None:
4887 probe._metadata = metadata
4888 size = sum(len(line) for line in traceback.format_exception_only(probe))
4889 # Python 3.12 draws ``end_offset - offset`` carets, unclamped.
4890 caret = 0
4891 if type(node.text) is str:
4892 caret = max(0, node.offset or 0, node.end_offset or 0)
4893 except Exception:
4894 return False
4895 return size <= 3 * verbatim + caret + _SYNTAX_ERROR_RENDER_SLACK
4898def _syntax_error_location_block(node: Any) -> str:
4899 """The ``File ...`` and source lines the traceback prints for *node*.
4901 ``TracebackException._format_syntax_error`` prints
4902 `` File "<filename>", line <lineno>`` (only when ``lineno`` is not
4903 ``None``; ``<string>`` for an empty or missing file name) and, on the
4904 next line, the source ``text`` without its trailing newlines and
4905 leading spaces, so a label ending one line and a value starting the
4906 next are adjacent text. Called only for fields of the canonical types
4907 (:data:`_SYNTAX_ERROR_FIELD_TYPES`).
4908 """
4909 lines = []
4910 if node.lineno is not None:
4911 lines.append(
4912 f' File "{node.filename or "<string>"}", line {node.lineno}'
4913 )
4914 if node.text is not None:
4915 lines.append(" " + node.text.rstrip("\n").lstrip(" \n\f"))
4916 return "\n".join(lines)
4919def _syntax_error_details_changed(node: Any) -> bool:
4920 """True when a ``SyntaxError``'s extra traceback lines need redacting.
4922 The traceback module renders ``filename``, ``lineno``, ``text`` and
4923 ``msg`` outside ``str(node)``. A field that is not ``None`` or of the
4924 exact type the interpreter fills it with
4925 (:data:`_SYNTAX_ERROR_FIELD_TYPES`: ``str`` for ``msg``, ``filename``
4926 and ``text``, ``int`` -- not ``bool`` -- for ``lineno``, ``offset``,
4927 ``end_lineno`` and ``end_offset``) makes the caller rebuild the node,
4928 and so does a ``text`` longer than ``_LOG_REDACTION_MAX_CHARS`` (which
4929 redaction would cut) or an ``int`` field outside the accepted ranges
4930 (:func:`_syntax_error_fields_out_of_range`). Otherwise each of ``msg``, ``filename``, ``lineno`` and ``text`` is
4931 checked on its own, and the ``File ...`` line joined with the source
4932 line below it (:func:`_syntax_error_location_block`). On Python 3.13+
4933 the keyword-typo suggestion may also replace the shown line with one
4934 from the source in ``_metadata``, or from the file ``filename`` names
4935 when that source is ``None``; that happens only when ``msg`` is
4936 ``"invalid syntax"`` or mentions a forgotten comma. Any of these that
4937 redacts differently, a source that is not an exact ``str``, or a file
4938 the sink would read makes the caller rebuild the node, which drops
4939 these details.
4940 """
4941 for name, kind in _SYNTAX_ERROR_FIELD_TYPES:
4942 value = getattr(node, name, None)
4943 if value is not None and type(value) is not kind:
4944 return True
4945 # Redaction cuts a longer ``text`` and appends a marker, so it changes
4946 # anyway; returning here also bounds what the range check encodes.
4947 if node.text is not None and len(node.text) > _LOG_REDACTION_MAX_CHARS: 4947 ↛ 4948line 4947 didn't jump to line 4948 because the condition on line 4947 was never true
4948 return True
4949 if _syntax_error_fields_out_of_range(node):
4950 return True
4951 for value in (node.msg, node.filename, node.lineno, node.text):
4952 if value is None:
4953 continue
4954 text = _safe_log_str(value)
4955 if _redact_exception_text(text) != text:
4956 return True
4957 block = _syntax_error_location_block(node)
4958 if _redact_exception_text(block) != block: 4958 ↛ 4959line 4958 didn't jump to line 4959 because the condition on line 4958 was never true
4959 return True
4960 metadata = getattr(node, "_metadata", None)
4961 if metadata is None:
4962 return False
4963 # Plain characters: a ``str`` subclass's own ``__ne__`` or
4964 # ``__contains__`` could answer differently here than in the sink.
4965 msg = _plain_text(node.msg)
4966 if not isinstance(node.text, str) or (
4967 isinstance(msg, str)
4968 and msg != "invalid syntax"
4969 and "Perhaps you forgot a comma" not in msg
4970 ):
4971 return False
4972 source = (
4973 metadata[2] if type(metadata) is tuple and len(metadata) == 3 else None
4974 )
4975 if type(source) is not str or len(source) > _LOG_REDACTION_MAX_CHARS: 4975 ↛ 4976line 4975 didn't jump to line 4976 because the condition on line 4975 was never true
4976 return True
4977 return any(
4978 _redact_exception_text(text) != text
4979 for text in (
4980 source,
4981 *source.splitlines(),
4982 *(line.lstrip() for line in source.splitlines()),
4983 )
4984 )
4987def redact_log_exception(record) -> None:
4988 """Give sinks a scrubbed exception graph without changing live exceptions.
4990 Loguru renders exception values, notes and group messages independently
4991 of record["message"]. Rebuild changed nodes using builtin exceptions only;
4992 custom constructors, copy hooks and group derive methods are never called.
4993 Custom non-group types retain their name in a RuntimeError message; group
4994 subclasses become builtin groups. Application-specific attributes are not
4995 copied to log output. Sensitive SyntaxError source details are omitted,
4996 as are those of a SyntaxError whose fields are not of the types the
4997 interpreter fills them with (:data:`_SYNTAX_ERROR_FIELD_TYPES`), not in
4998 the accepted ranges (:func:`_syntax_error_fields_out_of_range`), or whose
4999 rendering by the traceback module raises or runs long
5000 (:func:`_syntax_error_renders_bounded`).
5001 Tracebacks and safe notes are retained. Cycles and traversal limits become
5002 explicit placeholders instead of links back to unsanitized objects. An
5003 exception, cause, context, group member or note whose ``str()`` fails is
5004 rendered as ``<unprintable TypeName>`` rather than aborting the rebuild.
5005 An exception is kept as the original object only when its text was clean
5006 and no code its classes define, and no "Did you mean" suggestion, runs
5007 when the sinks render it again (:func:`_exception_text_is_stable` lists
5008 the conditions and what they do not cover) and its type's name joined
5009 with its text and notes (the ``TypeName: text`` header line, as the
5010 sinks render it, and the notes on the lines after it) is clean too;
5011 otherwise the sinks get a rebuilt node carrying exactly the text that
5012 was checked. The record's ``type`` stays the original class only when an
5013 ``enqueue=True`` sink can pickle it and its names are clean
5014 (:func:`_record_type_is_safe`); otherwise the top node is rebuilt even
5015 when it could be kept, and ``type`` becomes the rebuilt node's builtin
5016 type, so that sink never fails and dumps the unredacted original. A
5017 record with a ``type`` but no value (``None``) gets the same check: a
5018 ``type`` it rejects becomes ``RuntimeError``, with a value naming the
5019 original class (``RuntimeError: TypeName``).
5021 Application code the redaction itself runs (an exception's or a note
5022 object's ``__str__``, a ``__notes__`` or ``__cause__`` property) runs
5023 only while the graph is read (:func:`_plan_redacted_exception`), for
5024 every node, before anything is decided: such code can change any
5025 object, including one already read. Every decision -- which nodes are
5026 kept, and whether the record's ``type`` is -- is made afterwards, on
5027 the objects as they are then, by code that runs no application code
5028 for exception classes that do not hook the type, descriptor and
5029 instance-dict machinery those checks read through
5030 (:func:`_build_redacted_exception`), so for them nothing changes
5031 between those checks and the sinks. A class written to defeat this
5032 is outside the threat model (see the module docstring).
5033 """
5034 exc = record.get("exception") if record else None
5035 if not exc: 5035 ↛ 5036line 5035 didn't jump to line 5036 because the condition on line 5035 was never true
5036 return
5037 value = getattr(exc, "value", None)
5038 if value is None and isinstance(exc, tuple) and len(exc) >= 2:
5039 value = exc[1]
5041 kind = getattr(exc, "type", None)
5042 if kind is None and isinstance(exc, tuple) and len(exc) >= 1:
5043 kind = exc[0]
5044 if value is None:
5045 # No value to redact (``logger.opt(exception=(Type, None, tb))``),
5046 # but the sink still pickles ``type``: one it cannot keep is
5047 # replaced by a ``RuntimeError`` naming it, as the traceback module
5048 # renders a type with no value.
5049 if kind is None: 5049 ↛ 5050line 5049 didn't jump to line 5050 because the condition on line 5049 was never true
5050 return
5051 with _decision_phase():
5052 kind_is_safe = _record_type_is_safe(kind)
5053 if kind_is_safe:
5054 return
5055 new_value = RuntimeError(_named_text(kind, ""))
5056 traceback = getattr(exc, "traceback", None)
5057 if traceback is None and isinstance(exc, tuple) and len(exc) >= 3: 5057 ↛ 5059line 5057 didn't jump to line 5059 because the condition on line 5057 was always true
5058 traceback = exc[2]
5059 new_value.__traceback__ = traceback
5060 _replace_record_exception(record, exc, RuntimeError, new_value)
5061 return
5063 plans: dict = {}
5064 root = _plan_redacted_exception(
5065 value, plans, set(), [_LOG_EXCEPTION_NODE_BUDGET]
5066 )
5067 # Every step that can run application code has run. Everything below
5068 # reads the objects as they are now and runs builtin code only, so no
5069 # code can change a node, or the record's type, after it was checked.
5070 # Checks that depend on a class alone (its pickling, rendering hooks
5071 # and names, and ``sys.modules``'s keys) are answered once per class
5072 # for all of them (:func:`_once_per_phase`).
5073 with _decision_phase():
5074 kind_is_safe = _record_type_is_safe(kind)
5075 # A top node whose type cannot stay in the record
5076 # (:func:`_record_type_is_safe`) is rebuilt even when its text is
5077 # clean, so that its builtin type replaces it.
5078 new_value = _build_redacted_exception(
5079 root, plans, {}, force=None if kind_is_safe else root
5080 )
5081 if new_value is value:
5082 return
5083 # ``type`` stays the original class when it is safe to keep (the DB
5084 # row's prefix names it); otherwise it becomes the rebuilt value's
5085 # builtin type.
5086 new_kind = kind if kind_is_safe else type(new_value)
5087 _replace_record_exception(record, exc, new_kind, new_value)
5090def _replace_record_exception(
5091 record: Any, exc: Any, kind: Any, value: Any
5092) -> None:
5093 """Set *record*'s exception to *exc* with *kind* and *value* replaced."""
5094 if hasattr(exc, "_replace"):
5095 record["exception"] = exc._replace(type=kind, value=value)
5096 elif isinstance(exc, tuple) and len(exc) >= 3: 5096 ↛ 5099line 5096 didn't jump to line 5099 because the condition on line 5096 was always true
5097 record["exception"] = (kind, value, exc[2])
5098 else:
5099 record["exception"] = None
5102def _exception_redaction_failed(exc: Any) -> Any:
5103 """Replacement for a record exception whose redaction raised.
5105 Keeps the traceback (frames and source lines carry no exception text)
5106 and replaces the value with a fixed placeholder, so the record neither
5107 ships unredacted exception text nor silently loses where the error
5108 happened. ``None`` when even that cannot be built.
5109 """
5110 try:
5111 traceback = getattr(exc, "traceback", None)
5112 kind = getattr(exc, "type", None)
5113 name = _type_name(kind, "Exception")
5114 placeholder = RuntimeError(
5115 f"{name}: <exception text omitted: redaction failed>"
5116 )
5117 placeholder.__traceback__ = traceback
5118 if not _record_type_is_safe(kind):
5119 kind = RuntimeError
5120 return exc._replace(type=kind, value=placeholder, traceback=traceback)
5121 except Exception:
5122 return None
5125# ``BaseException``'s own ``__traceback__`` descriptor: the traceback a node
5126# carries, read without running anything its class defines.
5127_BASE_EXCEPTION_TRACEBACK = vars(BaseException)["__traceback__"]
5130@dataclasses.dataclass(eq=False)
5131class _ExceptionNodePlan:
5132 """What was read from one exception node and redacted, before any decision.
5134 :func:`_plan_redacted_exception` fills one per node; it is the pass
5135 that runs every step that can run application code.
5136 :func:`_build_redacted_exception` then decides, from this and from the
5137 node as it is by then, whether the original can be kept, and otherwise
5138 rebuilds it from the texts recorded here. A child is referred to by the
5139 ``id()`` of the original (a key of the plans, which hold the originals,
5140 so the ids stay unique), by a placeholder text, or by ``None``.
5141 """
5143 node: Any
5144 kind: Any
5145 is_group: bool
5146 cause: Any
5147 cause_ref: Any
5148 context: Any
5149 context_ref: Any
5150 suppress_context: Any
5151 subs: tuple
5152 sub_refs: list
5153 traceback: Any
5154 rendered_message: str
5155 message: str
5156 safe_message: str
5157 note_texts: list
5158 safe_notes: list
5159 # The texts read here were clean, as one would render them: printable,
5160 # unchanged by redaction, with notes that are a list or tuple of exact
5161 # ``str``. A node is kept only when this holds and it is unchanged.
5162 text_clean: bool
5165def _plan_redacted_exception(
5166 node: Any, plans: dict, in_progress: set, budget: list
5167) -> Any:
5168 """Read and redact the exception graph under *node*, deciding nothing.
5170 Every step of the redaction that can run application code runs here,
5171 for every node, before anything is decided: an exception's or a note
5172 object's ``str()``, a ``__notes__``, ``__cause__``, ``__context__``,
5173 ``exceptions`` or ``message`` that a class overrides, a group's
5174 message. That code can change any object, a node already read here
5175 included, so :func:`_build_redacted_exception` makes every decision
5176 afterwards, against the objects as they are by then. Each node's texts
5177 are read once and recorded (:class:`_ExceptionNodePlan`), and a node
5178 that is rebuilt carries exactly those texts.
5180 Returns the reference to *node*'s plan: its ``id()``, a placeholder
5181 text (the traversal budget ran out, which bounds both depth and group
5182 width, or the node is its own ancestor), or ``None``.
5183 """
5184 if node is None:
5185 return None
5186 if budget[0] <= 0:
5187 return "<exception traversal limit reached>"
5188 # Cached children still consume work and space in a group's output.
5189 budget[0] -= 1
5190 key = id(node)
5191 if key in plans:
5192 return key
5193 if key in in_progress:
5194 return "<exception cycle omitted>"
5195 in_progress.add(key)
5196 try:
5197 cause = node.__cause__
5198 cause_ref = _plan_redacted_exception(cause, plans, in_progress, budget)
5199 suppress_context = node.__suppress_context__
5200 # Not only when there is no cause: the sinks render each exception
5201 # once, so a cause already rendered elsewhere in the graph (shared
5202 # by two group members) is skipped, and an unsuppressed context is
5203 # rendered in its place.
5204 context = node.__context__ if not suppress_context else None
5205 context_ref = _plan_redacted_exception(
5206 context, plans, in_progress, budget
5207 )
5209 is_group = isinstance(node, BaseExceptionGroup)
5210 subs = tuple(node.exceptions) if is_group else ()
5211 sub_refs: list = []
5212 for child in subs:
5213 if budget[0] <= 0:
5214 sub_refs.append("<remaining exceptions omitted>")
5215 break
5216 sub_refs.append(
5217 _plan_redacted_exception(child, plans, in_progress, budget)
5218 )
5220 # The text is read once here and, if the node is rebuilt, the
5221 # rebuilt node carries exactly what was checked. A ``str()`` that
5222 # raised (the placeholder below is not what a sink would render)
5223 # rules out keeping the original.
5224 try:
5225 rendered_message = _exact_str(str(node))
5226 printable = True
5227 except Exception:
5228 rendered_message = _unprintable_placeholder(node)
5229 printable = False
5230 if is_group:
5231 try:
5232 message = node.message
5233 except Exception:
5234 message = None
5235 if isinstance(message, str): 5235 ↛ 5240line 5235 didn't jump to line 5240 because the condition on line 5235 was always true
5236 # A ``str`` subclass compares and renders on its own terms;
5237 # check, compare and rebuild with its plain characters.
5238 message = _exact_str(message)
5239 else:
5240 message = _unprintable_placeholder(node)
5241 elif printable and issubclass(type(node), SyntaxError):
5242 # The header line shows ``msg``, not ``str(node)`` with its
5243 # `` (file, line N)`` suffix (:func:`_syntax_error_header_text`):
5244 # a rebuilt node carries ``msg`` alone (the file and line are
5245 # details it drops), redacted as one with the notes printed
5246 # right after it.
5247 try:
5248 message = _syntax_error_header_text(node, location=False)
5249 except Exception:
5250 message = rendered_message
5251 else:
5252 message = rendered_message
5253 safe_message = _redact_exception_text(message)
5254 # A group renders ``str(node)`` (message plus sub-exception count),
5255 # and a ``SyntaxError``'s ``str()`` (which the DB row renders) is
5256 # not its header text; for any other exception ``str(node)`` is
5257 # ``message`` itself, so there this check would only repeat
5258 # ``safe_message != message`` below: the guard only saves that
5259 # second redaction pass (dropping it is an equivalent mutant, not a
5260 # missing check).
5261 rendered_changed = (
5262 is_group or message is not rendered_message
5263 ) and _redact_exception_text(rendered_message) != rendered_message
5264 # A ``__notes__`` property that raises: the traceback module reports
5265 # it with the error's repr ("Ignored error getting __notes__: ..."),
5266 # which is not redacted, so the node is rebuilt with a placeholder
5267 # note instead (the placeholder alone marks the node as changed).
5268 # The same placeholder covers notes that raise while being iterated.
5269 notes_failed = False
5270 try:
5271 note_texts, notes_exact = _rendered_exception_notes(node)
5272 except Exception:
5273 notes_failed = True
5274 note_texts, notes_exact = [], True
5275 safe_notes = [_redact_exception_text(note) for note in note_texts]
5276 if notes_failed:
5277 safe_notes = ["<exception notes omitted: __notes__ raised>"]
5278 plans[key] = _ExceptionNodePlan(
5279 node=node,
5280 kind=type(node),
5281 is_group=is_group,
5282 cause=cause,
5283 cause_ref=cause_ref,
5284 context=context,
5285 context_ref=context_ref,
5286 suppress_context=suppress_context,
5287 subs=subs,
5288 sub_refs=sub_refs,
5289 traceback=_BASE_EXCEPTION_TRACEBACK.__get__(node),
5290 rendered_message=rendered_message,
5291 message=message,
5292 safe_message=safe_message,
5293 note_texts=note_texts,
5294 safe_notes=safe_notes,
5295 text_clean=(
5296 printable
5297 and not notes_failed
5298 and notes_exact
5299 and safe_message == message
5300 and not rendered_changed
5301 and safe_notes == note_texts
5302 ),
5303 )
5304 return key
5305 finally:
5306 in_progress.discard(key)
5309def _build_redacted_exception(
5310 ref: Any, plans: dict, built: dict, force: Any = None
5311) -> Any:
5312 """The graph the sinks get for plan *ref*: originals kept, the rest rebuilt.
5314 Runs after :func:`_plan_redacted_exception` has read the whole graph,
5315 and runs no application code itself for exception classes and objects
5316 that do not hook the machinery it reads through (see the module
5317 docstring's threat model), so for those nothing can change a node
5318 between the decision to keep it and the sinks rendering it. A node
5319 retains identity only when its recorded text was clean, it is still
5320 the node that text was read from (:func:`_node_is_kept_as_planned`),
5321 and every chained exception and member it links to is kept too; any
5322 other node is rebuilt from the texts recorded for it, so the text,
5323 notes and chained exceptions a sink renders for it are the ones checked
5324 (a node for which the traceback module would compute a "Did you mean"
5325 suggestion is rebuilt, so no suggestion is rendered for it). Rebuilt
5326 nodes are builtin exceptions (a class counts as builtin when
5327 :mod:`builtins` holds it under its name): application constructors,
5328 copy hooks and group derive methods are never called. Omitted branches (traversal
5329 budget, cycles) become placeholders. *force* is the reference of a node
5330 rebuilt even when it could be kept (see :func:`redact_log_exception`);
5331 its chained exceptions and members are not forced.
5332 """
5333 if ref is None:
5334 return None
5335 if type(ref) is str:
5336 return RuntimeError(ref)
5337 if ref in built:
5338 return built[ref]
5339 plan = plans[ref]
5340 new_cause = _build_redacted_exception(plan.cause_ref, plans, built)
5341 new_context = _build_redacted_exception(plan.context_ref, plans, built)
5342 new_subs = [
5343 _build_redacted_exception(sub_ref, plans, built)
5344 for sub_ref in plan.sub_refs
5345 ]
5346 keep = (
5347 ref != force
5348 and new_cause is plan.cause
5349 and new_context is plan.context
5350 and len(new_subs) == len(plan.subs)
5351 and all(new is old for new, old in zip(new_subs, plan.subs))
5352 and _node_is_kept_as_planned(plan)
5353 )
5354 if keep:
5355 new_node = plan.node
5356 else:
5357 new_node = _rebuilt_exception(plan, new_cause, new_context, new_subs)
5358 built[ref] = new_node
5359 return new_node
5362def _exact_note_texts(node: Any) -> Optional[list]:
5363 """*node*'s notes when they are a ``list`` or ``tuple`` of exact ``str``.
5365 ``[]`` for no notes, ``None`` for notes of any other form. Called only
5366 for a node :func:`_exception_text_is_stable` accepts, for which the
5367 lookup and the iteration run builtin code only.
5368 """
5369 notes = getattr(node, "__notes__", None)
5370 if notes is None:
5371 return []
5372 if not _type_is_one_of(type(notes), (list, tuple)): 5372 ↛ 5373line 5372 didn't jump to line 5373 because the condition on line 5372 was never true
5373 return None
5374 items = list(notes)
5375 if not all(type(item) is str for item in items): 5375 ↛ 5376line 5375 didn't jump to line 5376 because the condition on line 5375 was never true
5376 return None
5377 return items
5380def _node_is_kept_as_planned(plan: _ExceptionNodePlan) -> bool:
5381 """True when *plan*'s original node may go to the sinks as it is now.
5383 Its recorded texts were clean, and it is still what they were read
5384 from: the same class; its rendering and pickling still run no code of
5385 its own classes (:func:`_exception_text_is_stable`, checked first, by
5386 code that runs none of the node's); and its ``str()``, notes, chained
5387 exceptions, suppress flag, a group's message and members, a
5388 ``SyntaxError``'s rendered details, and its type's name joined with
5389 its text and notes (:func:`_type_header_changed`) are as recorded and
5390 clean. Application code that ran while the graph was read (a parent's
5391 or a note object's ``__str__``, say) may have changed any of these
5392 after the node itself was read. Once the stability check passes, every
5393 read and call below runs builtin code only, so nothing changes between
5394 this check and the sinks -- for classes and objects that do not hook
5395 the machinery these checks read through; the module docstring's threat
5396 model says why one that does is not covered.
5397 """
5398 if not plan.text_clean:
5399 return False
5400 node = plan.node
5401 try:
5402 if type(node) is not plan.kind:
5403 return False
5404 if not _exception_text_is_stable(node, plan.is_group):
5405 return False
5406 if node.__cause__ is not plan.cause:
5407 return False
5408 suppress_context = node.__suppress_context__
5409 if suppress_context is not plan.suppress_context: 5409 ↛ 5410line 5409 didn't jump to line 5410 because the condition on line 5409 was never true
5410 return False
5411 context = None if suppress_context else node.__context__
5412 if context is not plan.context: 5412 ↛ 5413line 5412 didn't jump to line 5413 because the condition on line 5412 was never true
5413 return False
5414 if plan.is_group:
5415 exceptions = node.exceptions
5416 if len(exceptions) != len(plan.subs) or not all( 5416 ↛ 5419line 5416 didn't jump to line 5419 because the condition on line 5416 was never true
5417 live is old for live, old in zip(exceptions, plan.subs)
5418 ):
5419 return False
5420 if node.message != plan.message: 5420 ↛ 5421line 5420 didn't jump to line 5421 because the condition on line 5420 was never true
5421 return False
5422 rendered = str(node)
5423 if type(rendered) is not str or rendered != plan.rendered_message:
5424 return False
5425 if _exact_note_texts(node) != plan.note_texts:
5426 return False
5427 is_syntax_error = issubclass(plan.kind, SyntaxError)
5428 if is_syntax_error:
5429 if _syntax_error_header_text(node, location=False) != plan.message: 5429 ↛ 5430line 5429 didn't jump to line 5430 because the condition on line 5429 was never true
5430 return False
5431 if _syntax_error_details_changed(node):
5432 return False
5433 # The sinks render the type's name, the text and the notes joined,
5434 # so a credential can span them; checked last, as it costs one more
5435 # redaction pass per name form (:func:`_rendered_type_prefixes`: one
5436 # for a builtin class, two for one defined at module level, three
5437 # for a nested one) and header form (two for an empty text, up to
5438 # three for a ``SyntaxError``, one otherwise).
5439 if _type_header_changed(
5440 plan.kind,
5441 rendered,
5442 plan.note_texts,
5443 _rendered_header_tails(node, rendered),
5444 ):
5445 return False
5446 # Rendered as the sinks will, last: the caret line is drawn from the
5447 # offsets, and a rendering that raises or runs long makes the sinks'
5448 # error handler print the exception being handled.
5449 if is_syntax_error and not _syntax_error_renders_bounded(node):
5450 return False
5451 except Exception:
5452 return False
5453 return True
5456def _rebuilt_exception(
5457 plan: _ExceptionNodePlan, new_cause: Any, new_context: Any, new_subs: list
5458) -> BaseException:
5459 """A builtin exception carrying the texts recorded in *plan*."""
5460 kind = plan.kind
5461 safe_message = plan.safe_message
5462 if plan.is_group:
5463 # The builtin constructor rebuilds the read-only message and
5464 # members, without trusting a subclass's derive implementation.
5465 # The type's name is read through ``type``'s descriptor, and
5466 # redacted (:func:`_type_name`).
5467 if not _is_builtin_class(kind):
5468 # Keep a group subclass's name visible in the rebuilt group.
5469 safe_message = _redact_exception_text(
5470 f"{_type_name(kind)}: {safe_message}"
5471 )
5472 new_node = BaseExceptionGroup(safe_message, new_subs)
5473 elif _is_builtin_class(kind):
5474 try:
5475 new_node = kind(safe_message)
5476 except Exception:
5477 new_node = RuntimeError(_named_text(kind, safe_message))
5478 else:
5479 new_node = RuntimeError(_named_text(kind, safe_message))
5480 new_node.__traceback__ = plan.traceback
5481 new_node.__cause__ = new_cause
5482 new_node.__context__ = new_context
5483 new_node.__suppress_context__ = plan.suppress_context
5484 safe_notes = plan.safe_notes
5485 if safe_notes:
5486 # Each note was redacted on its own; a credential can still span
5487 # the rebuilt header line and the notes.
5488 header = (
5489 _TYPE_NAME.__get__(type(new_node))
5490 + _rendered_header_tails(new_node, _exact_str(str(new_node)))[0]
5491 )
5492 safe_notes = _notes_fitting_header(header, safe_notes)
5493 if safe_notes:
5494 new_node.__notes__ = safe_notes
5495 return new_node
5498def redact_log_record(record) -> None:
5499 """loguru patcher: control-char strip + credential redaction of a record.
5501 Scrubs the message and bound ``extra`` values (:func:`_redact_log_record_text`,
5502 :func:`redact_log_extra`) and the rendered exception graph
5503 (:func:`redact_log_exception`), so a credential that reaches a record
5504 without going through ``scrub_error`` first -- bare ``field=value`` in
5505 the message, a secret bound via ``logger.bind()``, or one embedded in an
5506 exception's text -- is caught before any sink renders it. Kept here,
5507 rather than in ``log_utils``, because that module imports the web stack
5508 at module scope and the MCP subprocess (``mcp/server.py``'s
5509 ``configure_mcp_logging``) must not pull that in: this is the patcher
5510 both processes install (the web process through
5511 ``utilities.log_utils._sanitize_record``).
5513 Fail-closed per field: a field that fails to redact is replaced rather
5514 than passed through as-is. Routing keys in ``extra``
5515 (:data:`_LOG_STRUCTURAL_EXTRA_KEYS`) are preserved so a failed field
5516 does not change the destination of a database log record. An exception
5517 that fails to redact keeps its traceback with a placeholder value.
5519 The result is always picklable: an ``enqueue=True`` sink pickles the
5520 record, and a pickling failure makes loguru print the original exception
5521 to stderr. Non-plain ``extra`` values and unsafe exception types are
5522 replaced before reaching a sink. Kept exception nodes are checked for
5523 pickling hooks that could change later rendering, within the threat
5524 model in the module docstring.
5526 The message and every ``extra`` string leaf use
5527 :func:`_redact_log_record_text`, which adds the unprefixed ``Api-Key:``
5528 header, slash-bearing DSN password, separator-delimited credential
5529 parameter, and quoted credential-key patterns after the ordinary
5530 redaction passes. Exception values use :func:`redact_log_exception`,
5531 which applies the patterns to rendered fields too; the database row's
5532 exception prefix applies them separately. Application code invoked while
5533 reading an exception graph runs before the keep decisions and
5534 picklability checks.
5535 """
5536 try:
5537 # Preserve credential boundaries that control stripping may remove,
5538 # then catch names that become recognizable after normalization.
5539 record["message"] = _redact_log_record_text(record["message"])
5540 except Exception:
5541 record["message"] = "<log message redacted: sanitization error>"
5543 extra = record.get("extra")
5544 if extra:
5545 try:
5546 extra = redact_log_extra(extra)
5547 except Exception:
5548 extra = _structural_extra_only(extra)
5549 record["extra"] = extra
5550 else:
5551 extra = None
5553 exc = record.get("exception")
5554 if exc:
5555 try:
5556 redact_log_exception(record)
5557 except Exception:
5558 record["exception"] = _exception_redaction_failed(exc)
5560 if extra is not None:
5561 # Last, after the exception's redaction, which runs application code
5562 # (:func:`redact_log_exception`) that could change a value passed
5563 # through above (a routing key's); this check runs none.
5564 record["extra"] = _picklable_extra(extra)
5567def _structural_extra_only(extra: Any) -> dict:
5568 """The routing keys of *extra* alone; ``{}`` if even that fails.
5570 Taken from the keys *extra* holds, by :func:`_is_structural_extra_key`,
5571 rather than looked up by name: ``extra["username"]`` would also find a
5572 ``str`` subclass key that only compares equal to ``"username"``.
5573 """
5574 try:
5575 return {
5576 key: value
5577 for key, value in dict.items(extra)
5578 if _is_structural_extra_key(key)
5579 }
5580 except Exception:
5581 return {}