Coverage for src/local_deep_research/security/url_redaction.py: 95%
118 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Redact credentials embedded in a configured URL before it is logged.
3Operators sometimes put an API key straight into an LLM base URL -- as
4userinfo (``https://user:key@host/v1``), as a query or fragment parameter
5(``?api_key=...``, ``#access_token=...``), as a ``;`` path parameter
6(``/v1;api_key=...``), or as a bare path segment (``/sk-.../v1``).
7:func:`redact_url_credentials` removes or masks those while keeping the URL
8legible for troubleshooting.
10Dependency-free on purpose (``re`` / ``urllib`` only), so provider, router
11and error-handling modules can all import it without an import cycle.
12"""
14from __future__ import annotations
16import re
17from urllib.parse import unquote_plus, urlsplit, urlunsplit
19REDACTED = "REDACTED"
21# A parameter NAME is a credential name when any of its PARTS is. The
22# parts of a PHP/Rails-style array name are the text around its ``[`` /
23# ``]`` brackets (``api_key[0]`` -> ``api_key``; ``auth[token]`` ->
24# ``auth``, ``token``; ``api_key[prod]`` -> ``api_key``, ``prod``); an
25# all-digit or empty part (an index) is ignored, and a stray bracket
26# (``api_key]``) simply ends a part. A part is normalised by splitting
27# camelCase (``accessToken`` -> ``access_token``, ``XAPIKey`` ->
28# ``XAPI_Key``), lower-casing, and splitting into words on every character
29# that is not an ASCII letter or digit (``_``, ``-``, ``.``, whitespace --
30# ``api+key`` decodes to ``api key`` -- and also ``%00``, ``:`` and the
31# like). A part is a credential name when EITHER
32# * its last word is in ``_CREDENTIAL_SUFFIX_WORDS`` (so ``api_key``,
33# ``x-api-key``, ``subscription-key``, ``access_token``, ``authToken``,
34# ``clientSecret``, ``x-amz-signature``, ``db.password``,
35# ``passphrase``, and the OAuth grant secrets ``assertion``,
36# ``client_assertion`` (RFC 7521/7523), ``code_verifier`` (RFC 7636)
37# and ``oauth_verifier`` (RFC 5849) all match), OR
38# * its last word is a run-together compound: it ends in one of
39# ``_RUN_TOGETHER_SUFFIXES`` (``authtoken``, ``APITOKEN``,
40# ``clientsecret``, ``clientassertion``, ``dbpassword``,
41# ``keypassphrase``) or in one of the ``...key`` words of
42# ``_RUN_TOGETHER_KEY_NAMES`` (``apikey``, ``xapikey``, ``secretkey``,
43# ``privatekey``). ``key`` itself is NOT a run-together suffix, because
44# ordinary words end in it (``monkey``, ``turkey``, ``hockey``); a
45# ``...key`` compound outside that list (``fookey``) is not matched, OR
46# * its words, joined without separators, are one of
47# ``_CREDENTIAL_EXACT_NAMES`` -- credential names that cannot be
48# matched by their ending without also matching ordinary names:
49# ``code`` (``error_code``), ``auth`` (``oauth``), ``refresh``
50# (``force_refresh``), ``device_code`` (RFC 8628; ``country_code``),
51# ``id_token_hint`` (it carries an OpenID Connect ID token;
52# ``login_hint``) and the OAuth authorization code ``auth_code`` /
53# ``authorization_code`` (RFC 6749). Joining makes ``device_code``,
54# ``deviceCode`` and the run-together ``devicecode`` / ``DEVICECODE``
55# one name.
56# Each of these tests is also run with a trailing number or version taken
57# off the part -- final words that are digits or ``v`` + digits are
58# dropped, then trailing digits (and a ``v`` just before them) come off the
59# last word -- and once more with a plural ``s`` also taken off the last
60# word (not when it ends in ``ss``). So ``api_key_2``, ``apiKey2``,
61# ``key1``, ``token2``, ``access_token_v2``, ``x-api-key-v2`` and
62# ``api_keys`` match, while ``page2``, ``items`` and ``v2`` do not; this
63# only ever adds matches.
64# Other words match only as a whole word, so ``tokenizer``, ``passthrough``,
65# ``monkey`` and ``keyboard`` never match. Errs towards over-redaction: a
66# non-secret ``sort_key``, ``sort_keys``, ``max_tokens`` or
67# ``api_key[prod]`` is redacted too, and a name longer than
68# ``_MAX_PARAM_NAME_LEN`` is treated as a credential name without being
69# inspected (fail closed, and bounded work).
70_CREDENTIAL_SUFFIX_WORDS = frozenset(
71 {
72 "key",
73 "apikey",
74 "token",
75 "secret",
76 "password",
77 "passwd",
78 "pwd",
79 "pass",
80 "passphrase",
81 "signature",
82 "sig",
83 "credential",
84 "credentials",
85 "jwt",
86 "session",
87 "assertion",
88 "verifier",
89 }
90)
91_RUN_TOGETHER_SUFFIXES = (
92 "token",
93 "secret",
94 "password",
95 "passwd",
96 "passphrase",
97 "assertion",
98 "verifier",
99)
100_RUN_TOGETHER_KEY_NAMES = (
101 "apikey",
102 "appkey",
103 "accesskey",
104 "authkey",
105 "clientkey",
106 "encryptionkey",
107 "licensekey",
108 "masterkey",
109 "passkey",
110 "privatekey",
111 "secretkey",
112 "sessionkey",
113 "signingkey",
114 "subscriptionkey",
115)
116# Stored without separators: compared with a part's words joined by "".
117_CREDENTIAL_EXACT_NAMES = frozenset(
118 name.replace("_", "")
119 for name in (
120 "sas",
121 "auth",
122 "authorization",
123 "bearer",
124 "code",
125 "refresh",
126 "device_code",
127 "id_token_hint",
128 "auth_code",
129 "authorization_code",
130 )
131)
132# A final name word that is a number or a version (``2``, ``v2``).
133_NUMBER_OR_VERSION_WORD_RE = re.compile(r"v?[0-9]+")
134# Longest parameter name inspected word by word; a longer one is treated
135# as a credential name. Real parameter names are far shorter.
136_MAX_PARAM_NAME_LEN = 256
137# Longest value :func:`redact_url_credentials` parses; a longer one is
138# logged as ``<unknown>``. A configured base URL is far shorter; the cap
139# bounds the work done in (often unconditional) log lines on a value a
140# user can supply.
141MAX_REDACTABLE_URL_LEN = 4096
142_CAMEL_BOUNDARY_RE = re.compile(
143 r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])"
144)
145_NAME_WORD_SEPARATOR_RE = re.compile(r"[^a-z0-9]+")
146_ARRAY_BRACKET_RE = re.compile(r"[\[\]]")
147# ``?`` is a separator too: ``?x=1?api_key=...`` (a repeated ``?``) and a
148# value that embeds a URL (``next=https://h/p?token=...``) both carry their
149# credential after a ``?`` that ``urlsplit`` leaves inside the query.
150_PARAM_SEPARATORS_RE = re.compile(r"([&;?])")
151# A value-less query/fragment item at least this long (``?sk-...``) is
152# assumed to be a bare credential.
153_BARE_ITEM_REDACT_MIN_LEN = 17
154# Path segments shaped like a provider key:
155# * ``sk-`` / ``key-`` in any letter case (``SK-``, ``Key-``) + 16 or
156# more base64url characters, or
157# * 32+ characters of ``[A-Za-z0-9_]`` -- NO ``-`` -- containing both a
158# letter and a digit.
159# Excluding ``-`` keeps hyphenated model and deployment names
160# (``text-embedding-3-large-2024-preview-model``, UUIDs) legible. The
161# trade-offs: a dash-bearing key without an ``sk-``/``key-`` prefix (some
162# Google keys) is not caught, and a 32-hex id such as a Cloudflare account
163# id IS redacted (over-redaction, the safe direction).
164_KEY_LIKE_SEGMENT_RE = re.compile(
165 r"^(?:(?:sk|key)-[A-Za-z0-9_\-]{16,}"
166 r"|(?=[A-Za-z0-9_]*[A-Za-z])(?=[A-Za-z0-9_]*[0-9])[A-Za-z0-9_]{32,})$",
167 # ASCII: without it, IGNORECASE lets the classes match four non-ASCII
168 # letters too (U+0130, U+0131, U+017F, U+212A).
169 re.IGNORECASE | re.ASCII,
170)
171_PORT_RE = re.compile(r"[0-9]+")
172# Characters that never belong in a host we log: a ``;`` / ``&`` / ``=``
173# after the host means parameters (``https://host;api_key=...``) ended up
174# inside the parsed authority, where they would not be redacted.
175_AUTHORITY_PARAM_CHARS_RE = re.compile(r"[;&=]")
176# End of a scheme-less value's leading authority-like part.
177_AUTHORITY_END_RE = re.compile(r"[/?#]")
178_HTTP_SCHEMES = frozenset({"http", "https"})
181def _name_variants(words: list[str]) -> list[list[str]]:
182 """``words`` as given, without a trailing number or version, and
183 without that and a plural ``s`` (see the rule above). A last word that
184 the stripping would leave empty (``v2``, ``s``) is kept as it was."""
185 trimmed = list(words)
186 while len(trimmed) > 1 and _NUMBER_OR_VERSION_WORD_RE.fullmatch(
187 trimmed[-1]
188 ):
189 trimmed.pop()
190 last = trimmed[-1]
191 stem = last.rstrip("0123456789")
192 if stem != last and stem.endswith("v"):
193 stem = stem[:-1]
194 if stem:
195 trimmed[-1] = stem
196 variants = [words, trimmed]
197 last = trimmed[-1]
198 if len(last) > 1 and last.endswith("s") and not last.endswith("ss"):
199 variants.append([*trimmed[:-1], last[:-1]])
200 return variants
203def _words_name_a_credential(words: list[str]) -> bool:
204 """The three tests of the rule above, for one spelling of a part."""
205 if "".join(words) in _CREDENTIAL_EXACT_NAMES:
206 return True
207 last = words[-1]
208 return (
209 last in _CREDENTIAL_SUFFIX_WORDS
210 or last.endswith(_RUN_TOGETHER_SUFFIXES)
211 or last.endswith(_RUN_TOGETHER_KEY_NAMES)
212 )
215def _is_credential_name_part(part: str) -> bool:
216 """The rule above for one bracket-free part of a parameter name."""
217 words = [
218 w
219 for w in _NAME_WORD_SEPARATOR_RE.split(
220 _CAMEL_BOUNDARY_RE.sub("_", part).lower()
221 )
222 if w
223 ]
224 if not words: 224 ↛ 225line 224 didn't jump to line 225 because the condition on line 224 was never true
225 return False
226 return any(_words_name_a_credential(v) for v in _name_variants(words))
229def is_credential_param_name(raw_name: str) -> bool:
230 """Return True when a URL parameter name names a credential.
232 Linear in the length of the name, and a name longer than
233 ``_MAX_PARAM_NAME_LEN`` is reported
234 as a credential name without further inspection."""
235 if len(raw_name) > _MAX_PARAM_NAME_LEN:
236 return True
237 return any(
238 _is_credential_name_part(part)
239 for part in _ARRAY_BRACKET_RE.split(unquote_plus(raw_name))
240 if part.strip() and not part.strip().isdigit()
241 )
244def _redact_param_list(params: str, *, redact_bare: bool) -> str:
245 """Redact credential values in a ``&`` / ``;`` / ``?`` separated list,
246 keeping every other parameter and separator unchanged."""
247 if not params:
248 return params
249 out = []
250 for piece in _PARAM_SEPARATORS_RE.split(params):
251 if piece in ("&", ";", "?"):
252 out.append(piece)
253 continue
254 name, sep, _value = piece.partition("=")
255 if sep and is_credential_param_name(name):
256 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: URL parameter text
257 piece = f"{name}={REDACTED}"
258 elif (
259 not sep and redact_bare and len(piece) >= _BARE_ITEM_REDACT_MIN_LEN
260 ):
261 piece = REDACTED
262 out.append(piece)
263 return "".join(out)
266def _redact_path(path: str) -> str:
267 """Redact ``;name=value`` path parameters and key-shaped segments."""
268 segments = []
269 for segment in path.split("/"):
270 head, sep, params = segment.partition(";")
271 if _KEY_LIKE_SEGMENT_RE.match(head):
272 head = REDACTED
273 if sep:
274 params = _redact_param_list(params, redact_bare=False)
275 segments.append(f"{head}{sep}{params}")
276 return "/".join(segments)
279def _drop_schemeless_userinfo(url: str) -> str:
280 """``user:secret@host/v1`` (no ``//``) parses with ``user`` as the
281 scheme and the secret in the path; drop everything up to the ``@`` of
282 the leading authority-like part, which ends at the first ``/``, ``?``
283 or ``#``. An ``@`` only later in the value (a password containing one
284 of those such as ``user:Abc/123@host``, or an ``@`` in the query or
285 fragment such as ``host?x=a@b``) is left for the caller's fail-closed
286 ``@`` check."""
287 head = _AUTHORITY_END_RE.split(url, maxsplit=1)[0]
288 if "@" in head:
289 return url[head.rindex("@") + 1 :]
290 return url
293def _authority_is_well_formed(authority: str) -> bool:
294 """True when ``host[:port]`` (or ``[v6][:port]``) has an empty or
295 all-digit port. A non-digit "port" means ``urlsplit`` cut the
296 authority short -- typically at a ``/``, ``?`` or ``#`` inside an
297 unencoded password -- so the real userinfo was never isolated. Also
298 False when the authority holds a ``;``, ``&`` or ``=``
299 (``host;api_key=...``): parameters there are not redacted."""
300 if _AUTHORITY_PARAM_CHARS_RE.search(authority):
301 return False
302 if authority.startswith("["):
303 close = authority.find("]")
304 if close == -1: 304 ↛ 305line 304 didn't jump to line 305 because the condition on line 304 was never true
305 return False
306 rest = authority[close + 1 :]
307 if not rest: 307 ↛ 308line 307 didn't jump to line 308 because the condition on line 307 was never true
308 return True
309 if not rest.startswith(":"): 309 ↛ 310line 309 didn't jump to line 310 because the condition on line 309 was never true
310 return False
311 port = rest[1:]
312 else:
313 port = authority.rpartition(":")[2] if ":" in authority else ""
314 return not port or bool(_PORT_RE.fullmatch(port))
317def redact_url_credentials(url: object) -> str:
318 """Return ``url`` with embedded credentials removed, for logging.
320 * userinfo (``user:pass@``) is dropped -- also for a scheme-less
321 ``user:pass@host/v1``;
322 * in the query string, the fragment, and ``;`` path parameters, the
323 value of every credential-named parameter (see the name rule above;
324 ``&``, ``;`` and ``?`` all separate parameters) becomes ``REDACTED``;
325 * a value-less query or fragment item of 17+ characters (``?sk-...``)
326 becomes ``REDACTED``;
327 * a path segment shaped like a provider key (see
328 ``_KEY_LIKE_SEGMENT_RE``) becomes ``REDACTED``.
330 Fails closed -- returns ``"<unknown>"`` -- when the userinfo cannot be
331 isolated reliably: any ``@`` in the path, query or fragment (so the
332 count of ``@`` in the value differs from the count in the parsed
333 authority -- this includes a harmless ``?email=a@example.com``), or an
334 authority whose port is not all digits (an unencoded ``/``, ``?`` or
335 ``#`` in a password ends the authority early), or an authority that
336 contains ``;``, ``&`` or ``=`` (``https://host;api_key=...``).
337 Non-string and empty inputs, and any value longer than
338 ``MAX_REDACTABLE_URL_LEN`` characters, also return ``"<unknown>"``;
339 the function never raises, and its work is linear in the input.
341 The scheme is lower-cased (``urlsplit`` does that); the host (with its
342 original case) and the port are kept as given; the rest of the URL is
343 kept unchanged except for the replacements above (``urlsplit`` also
344 drops an empty trailing ``?`` or ``#``).
346 Limits: a credential under a name the rule does not recognise, a short
347 or non key-shaped path segment, and percent-encoded separators inside a
348 value are not detected.
349 """
350 if not isinstance(url, str) or not url or len(url) > MAX_REDACTABLE_URL_LEN:
351 return "<unknown>"
352 try:
353 parsed = urlsplit(url)
354 if not parsed.netloc and parsed.scheme.lower() not in _HTTP_SCHEMES:
355 stripped = _drop_schemeless_userinfo(url)
356 if stripped != url:
357 url = stripped
358 parsed = urlsplit(url)
359 except ValueError:
360 return "<unknown>"
361 if url.count("@") != parsed.netloc.count("@"):
362 # An @ outside the authority (path, query or fragment): either the
363 # userinfo was not isolated -- a password containing ``@`` and then
364 # ``/``, ``?`` or ``#`` splits across the authority boundary -- or a
365 # scheme-less value still carries one after stripping. Also covers
366 # an innocent ``?email=a@example.com``: withheld rather than guessed.
367 return "<unknown>"
368 redacted = parsed._replace(
369 path=_redact_path(parsed.path),
370 query=_redact_param_list(parsed.query, redact_bare=True),
371 fragment=_redact_param_list(parsed.fragment, redact_bare=True),
372 )
373 if parsed.netloc:
374 authority = parsed.netloc.rpartition("@")[2]
375 if not _authority_is_well_formed(authority):
376 return "<unknown>"
377 redacted = redacted._replace(netloc=authority)
378 elif redacted == parsed:
379 # Nothing to redact: pass a scheme-less value through as given.
380 return url or "<unknown>"
381 return urlunsplit(redacted) or "<unknown>"