Coverage for src/local_deep_research/utilities/openalex_enrichment.py: 98%
135 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Batch DOI → OpenAlex source_id enrichment.
4Resolves paper DOIs to OpenAlex source IDs in a single batch HTTP request
5(up to 50 DOIs per call). This populates ``openalex_source_id`` and
6``source_type`` on result dicts so the journal reputation filter can look
7up journals by ID rather than fragile name matching.
9Runs as a pre-enrichment layer before content filters — the existing
10tiered scoring system is unchanged. This just gives Tier 2 (OpenAlex
11snapshot lookup) a reliable key to work with.
12"""
14import unicodedata
15from typing import Any, Callable, Dict, List, Optional, Tuple
17from loguru import logger
19from ..constants import OPENALEX_ENRICHMENT_API_TIMEOUT, USER_AGENT
20from ..security.log_sanitizer import scrub_error
21from ..security.safe_requests import safe_get
22from .citation_normalizer import _extract_doi
25_OPENALEX_API = "https://api.openalex.org"
26_MAX_DOIS_PER_REQUEST = 50
28# HTTP statuses treated as "this credential was refused".
29#
30# Sourced, not assumed. OpenAlex's published error table
31# (https://help.openalex.org/api/errors/) documents 200, 301, 400,
32# 403 ("Forbidden - you don't have access to this resource"), 404, 429
33# and 500. It documents **no 401**, and
34# https://help.openalex.org/api/authentication/ does not state which
35# status a rejected key earns. So this is deliberately a small
36# *defensive* set: 403 is documented, and 401 is the standard status for
37# a refused bearer token. Nothing else is guessed at.
38#
39# In particular there is no separate "daily budget exceeded" status to
40# map: the errors page defines 429 as "Rate limit or daily credit budget
41# exceeded; slow down or wait for the reset", and the authentication
42# page confirms "Two things return 429 Too Many Requests: exceeding your
43# daily budget, or making more than 100 requests per second". 429 is
44# already handled by the engine's rate-limit branch, which points at the
45# free key that raises the daily budget 10x.
46#
47# A refused key must never look like "no results": it is an operator
48# misconfiguration. See :func:`send_with_key_fallback`.
49OPENALEX_AUTH_FAILURE_STATUSES = frozenset({401, 403})
52def normalize_openalex_api_key(api_key: Any) -> Optional[str]:
53 """Normalize a configured OpenAlex API key to a real key, or ``None``.
55 ``None`` means "make the request keyless" — OpenAlex works without a
56 key, so an unusable configured value must degrade to the free tier
57 rather than being sent as a ``Bearer`` token that earns a 401/403 and
58 turns a working engine into one that returns nothing.
60 Collapses to ``None``:
62 * any non-``str`` value (``True``/``1`` from a mis-typed setting used
63 to raise ``AttributeError`` on ``.strip()``),
64 * whitespace-only values (``None``, never ``""``),
65 * the ``"False"`` settings sentinel, case-insensitively,
66 * every placeholder ``BaseSearchEngine._is_valid_api_key`` rejects —
67 ``your-api-key-here``, ``<your_api_key>``, ``${OPENALEX_API_KEY}``, …
68 * any value that is not usable as an HTTP header value. Two
69 independent rules, both required:
71 - *internal whitespace or control characters*. Such a value cannot
72 be a real key, and a newline in it makes ``requests`` raise
73 ``InvalidHeader`` whose message embeds the value's ``repr``,
74 which neither scrub pass can match (the literal holds a real
75 newline, the message holds a backslash and an "n"). Rejecting it
76 keeps it out of the logs.
77 - *not latin-1 encodable*. This is the hard constraint: CPython's
78 ``http.client`` encodes every header value as latin-1
79 (``values[i] = one_value.encode('latin-1')``), and neither
80 ``requests``' own header validation nor anything below it gates
81 the encoding. A key carrying U+2010, U+2013 or U+2019 — what a
82 PDF paste or an editor's autocorrect produces — therefore raises
83 ``UnicodeEncodeError`` *before any request leaves the process*.
84 That is not an HTTP status, so :func:`send_with_key_fallback`
85 cannot turn it into a keyless retry by inspecting
86 ``response.status_code``; the value has to be refused here.
87 Characters that *are* latin-1 (``é``, ``ü``) are left alone —
88 they encode fine and could be part of a real key.
90 This lives here, rather than next to the placeholder list in
91 ``web_search_engines/search_engine_base.py``, because all three
92 OpenAlex callers need it — the search engine, the base-class DOI
93 enrichment pass, and the research-library downloader — and only this
94 module is low-level enough for the downloader to import. The
95 placeholder list itself is *not* duplicated: it is imported inside
96 the function, which keeps the *module-level* import graph free of the
97 engine package (``research_library.downloaders.openalex`` imports
98 this module at module level).
99 """
100 if not isinstance(api_key, str):
101 return None
103 candidate = api_key.strip()
104 # "False" is the repo-wide "unset" sentinel for string settings that
105 # round-tripped through a boolean UI control; it is not a placeholder
106 # name, so the shared list does not (and should not) carry it.
107 if candidate.lower() == "false":
108 return None
110 from ..web_search_engines.search_engine_base import (
111 _is_api_key_placeholder,
112 )
114 if _is_api_key_placeholder(candidate):
115 return None
117 # Checked after the placeholder list so a whitespace-bearing
118 # placeholder is still dropped silently rather than warned about.
119 # The warning deliberately does not include the value.
120 unusable_as_header = any(
121 character.isspace() or unicodedata.category(character).startswith("C")
122 for character in candidate
123 )
124 if not unusable_as_header:
125 # The encoding check the whitespace/control predicates do not
126 # cover. ``http.client`` encodes header values as latin-1 and
127 # nothing between here and there catches the resulting
128 # ``UnicodeEncodeError``, so a key that is not latin-1 encodable
129 # would take out every OpenAlex call — search, enrichment and
130 # download — with no status code for the keyless fallback to
131 # branch on. See the docstring.
132 try:
133 candidate.encode("latin-1")
134 except UnicodeEncodeError:
135 unusable_as_header = True
137 if unusable_as_header:
138 logger.warning(
139 "Ignoring the configured OpenAlex API key: it is not usable as "
140 "an HTTP header value (it contains whitespace, control "
141 "characters, or characters outside latin-1). Re-copy the key "
142 "from https://openalex.org/settings/api"
143 )
144 return None
145 return candidate
148ERROR_BODY_SCRUB_LIMIT = 8192
151def bounded_error_body(text: str, limit: int = ERROR_BODY_SCRUB_LIMIT) -> str:
152 """Cut an upstream error body down to a bounded, scrub-safe prefix.
154 ``safe_requests`` allows a 1 GiB response body, and running all of it
155 through the ~18 compiled patterns of the credential scrubbers only to
156 keep a couple of hundred characters is pointless work. This bounds
157 the *regex passes* — ``response.text`` has already decoded the whole
158 body by the time it is called, so it bounds no allocation.
160 Truncating before scrubbing is only safe if the cut cannot create a
161 partial credential, because the scrubbers are shape-anchored and a
162 scrub can *shrink* the text (``Bearer <token>`` collapses to 17
163 characters), pulling content from far beyond the caller's final cap
164 into the characters that get logged.
166 The invariant that makes it safe: :func:`normalize_openalex_api_key`
167 refuses any key containing whitespace, so a key is always contained
168 in a single whitespace-free run. Cutting back to the last whitespace
169 character at or before *limit* therefore drops the trailing partial
170 run in its entirety — no fragment of a token that straddles the
171 boundary can survive. A body with no whitespace at all in its first
172 *limit* characters yields ``""``: the whole prefix is one unbroken
173 run and none of it can be shown to be credential-free.
175 Args:
176 text: The decoded response body.
177 limit: Maximum number of characters considered before the cut
178 back to a whitespace boundary.
180 Returns:
181 *text* unchanged when it is no longer than *limit*, otherwise a
182 prefix ending at a whitespace character, possibly ``""``.
183 """
184 if len(text) <= limit:
185 return text
186 window = text[:limit]
187 for index in range(len(window) - 1, -1, -1):
188 if window[index].isspace():
189 return window[: index + 1]
190 return ""
193def send_with_key_fallback(
194 send_request: Callable[[bool], Any],
195 *,
196 api_key: Optional[str],
197 context: str,
198 on_key_rejected: Optional[Callable[[], None]] = None,
199 before_retry: Optional[Callable[[], None]] = None,
200) -> Tuple[Any, Optional[str]]:
201 """Send an OpenAlex request; drop a refused key and retry keyless once.
203 OpenAlex serves the same endpoints with no key at all, at a lower
204 daily budget (https://help.openalex.org/api/authentication/), so a
205 stale, rotated, typo'd or revoked key must never cost the caller its
206 results. Every LDR call site that can send the OpenAlex credential
207 goes through this one function so the guarantee cannot be
208 implemented in one place and forgotten in another.
210 On an auth failure (:data:`OPENALEX_AUTH_FAILURE_STATUSES`) *while a
211 key was actually sent*, or when the keyed send raises
212 ``UnicodeEncodeError`` (see below):
214 * exactly one warning is logged. It is scrub-safe by construction:
215 the only interpolated values are *context* (a caller literal) and
216 the integer status code — the key itself is never formatted into
217 it;
218 * the key is dropped. This function returns ``None`` as the key in
219 effect and calls *on_key_rejected* so the caller can clear any
220 cached ``Authorization`` header, which means no later request pays
221 another wasted round-trip with a credential already known to be
222 refused;
223 * *before_retry* runs (rate-limit accounting for the extra request);
224 * the same request is retried once, keyless.
226 When no key was sent, or the response is not an auth failure, the
227 response is returned untouched and nothing is logged — keyless
228 behaviour is exactly what it was before the key existed.
230 If the keyless retry fails too, its response is returned as-is and
231 the caller applies its own pre-existing failure handling.
233 ``UnicodeEncodeError`` from the keyed send is treated exactly like a
234 rejection. It is the one exception that can only come from encoding a
235 request *header value*: ``http.client`` encodes header values as
236 latin-1 *before* opening the connection, so no request went out and a
237 keyless resend is safe and correct. (URLs and query parameters are
238 percent-encoded by ``requests`` and raise ``InvalidURL``, not
239 ``UnicodeEncodeError``.) Which header is not knowable here.
240 :func:`normalize_openalex_api_key` already refuses non-latin-1 keys,
241 so on every in-tree path the ``Authorization`` value is already
242 known-encodable and the likelier source is another configured header
243 value — the contact email interpolated into the ``User-Agent``, which
244 ``requests`` emits before ``Authorization``. Dropping the key is
245 still the right response: it is the only credential-bearing header,
246 and a keyless resend costs nothing when the key was not at fault.
247 The catch is deliberately this one exception type: any other failure
248 is a real network or protocol error, and retrying it keyless would
249 both hide the fault and throw away a key the server never refused.
251 Args:
252 send_request: ``send_request(with_api_key)`` issues the request
253 and returns the response. It MUST omit the ``Authorization``
254 header when *with_api_key* is False.
255 api_key: The key that will be sent, or ``None`` for keyless.
256 context: Short literal naming the call site, for the log line.
257 on_key_rejected: Called once, before the retry, to clear the
258 caller's cached credential state.
259 before_retry: Called once, immediately before the retry.
261 Returns:
262 ``(response, api_key)`` where *api_key* is ``None`` once dropped.
263 """
265 def _drop_and_resend_keyless() -> Tuple[Any, Optional[str]]:
266 # on_key_rejected runs BEFORE the resend, so even if the keyless
267 # request raises, the refused key is already gone and no later
268 # call can send it again.
269 if on_key_rejected is not None: 269 ↛ 271line 269 didn't jump to line 271 because the condition on line 269 was always true
270 on_key_rejected()
271 if before_retry is not None:
272 before_retry()
273 return send_request(False), None
275 try:
276 response = send_request(bool(api_key))
277 except UnicodeEncodeError:
278 if not api_key:
279 # Nothing to drop and nothing to retry differently: the
280 # failure did not come from the credential.
281 raise
282 # The exception carries one character of the offending header
283 # value and its offset, so it is never interpolated into the log
284 # line.
285 logger.warning(
286 f"{context}: a configured OpenAlex request header value (the "
287 "API key or the contact email used in the User-Agent) cannot "
288 "be encoded as an HTTP header value; dropping the key and "
289 "continuing keyless. Re-check both settings at "
290 "https://openalex.org/settings/api"
291 )
292 return _drop_and_resend_keyless()
294 if not api_key:
295 return response, api_key
296 if response.status_code not in OPENALEX_AUTH_FAILURE_STATUSES:
297 return response, api_key
299 logger.warning(
300 f"{context}: OpenAlex rejected the configured API key "
301 f"(HTTP {response.status_code}); dropping it and continuing "
302 "keyless. Check or rotate the key at "
303 "https://openalex.org/settings/api"
304 )
305 return _drop_and_resend_keyless()
308def _normalize_doi(doi: str) -> str:
309 """Normalize a DOI to ``https://doi.org/<...>`` form for OpenAlex.
311 The ``startswith`` prefix checks below are fully anchored and
312 CodeQL-safe. A previous code-scanning bot comment cited alert 7635
313 (``py/incomplete-url-substring-sanitization``) against an earlier
314 snapshot of this file; the current CodeQL scan does not raise it,
315 and the anchored ``startswith`` pattern is the rule's recommended
316 mitigation. Refactoring to a bare-first normalization was
317 evaluated (PR #3081) but rejected as no-op churn — the
318 ``https://doi.org/`` form OpenAlex actually returns round-trips
319 unchanged through every branch here. Do not refactor without a
320 new, reproducible functional issue.
321 """
322 doi = doi.strip()
323 if doi.startswith("https://doi.org/"):
324 return doi
325 if doi.startswith("http://doi.org/"):
326 return doi.replace("http://", "https://")
327 if doi.startswith("10."):
328 return f"https://doi.org/{doi}"
329 return doi
332def enrich_results_with_source_ids(
333 results: List[Dict[str, Any]],
334 email: Optional[str] = None,
335 api_key: Optional[str] = None,
336 *,
337 on_key_rejected: Optional[Callable[[str], None]] = None,
338) -> List[Dict[str, Any]]:
339 """
340 Batch-enrich results with OpenAlex source_id by DOI lookup.
342 For results that have a DOI but no ``openalex_source_id``, makes a
343 single batch request to the OpenAlex works endpoint to resolve
344 DOI → journal/conference source_id.
346 The results list is modified in-place and also returned.
348 Args:
349 results: List of result dicts from search engines.
350 email: Optional email included in the User-Agent for identification.
351 api_key: Optional OpenAlex API key for a higher daily request budget.
352 Normalized through :func:`normalize_openalex_api_key`, so a
353 placeholder or sentinel value degrades to a keyless request
354 instead of a guaranteed 401. A real key OpenAlex refuses is
355 dropped for the rest of this call and every remaining batch
356 goes out keyless (:func:`send_with_key_fallback`), so a bad
357 key costs at most one extra request, never the enrichment.
358 on_key_rejected: Called with the refused key, once, the first
359 time OpenAlex refuses it. This call's own drop is a local,
360 so a caller that keeps the engine alive across runs passes
361 this to latch the rejection on itself — otherwise the next
362 run resolves the same key from the same settings snapshot
363 and pays the wasted round-trip again.
364 ``BaseSearchEngine._note_openalex_key_rejected`` is the
365 production implementation.
367 Returns:
368 The same results list, with ``openalex_source_id`` and
369 ``source_type`` injected where resolved.
370 """
371 if not results:
372 return results
374 api_key = normalize_openalex_api_key(api_key)
376 # Collect DOIs for results that need enrichment
377 doi_to_indices: Dict[str, List[int]] = {}
378 for i, result in enumerate(results):
379 # Skip results that already have a source_id
380 if result.get("openalex_source_id"):
381 continue
383 doi = _extract_doi(result)
384 if doi:
385 normalized = _normalize_doi(doi)
386 doi_to_indices.setdefault(normalized, []).append(i)
388 if not doi_to_indices:
389 logger.debug("DOI enrichment: no DOIs to resolve")
390 return results
392 # Batch DOIs into chunks of MAX_DOIS_PER_REQUEST
393 all_dois = list(doi_to_indices.keys())
394 enriched_count = 0
395 # Kept for redaction after the key is dropped: ``scrub_error`` can
396 # only redact a literal it is handed, and ``api_key`` becomes None
397 # the moment OpenAlex refuses it.
398 rejected_key: Optional[str] = None
400 def _drop_rejected_key() -> None:
401 # The single mutation point: this runs *before* the keyless
402 # retry, so even if that retry raises, the refused key is already
403 # gone and no later chunk can send it again.
404 nonlocal api_key, rejected_key
405 # ``or rejected_key``: a second call must not store None over the
406 # literal and drop it out of the redaction set.
407 rejected_key = api_key or rejected_key
408 api_key = None
409 if on_key_rejected is not None and rejected_key:
410 # Last, so a raising callback cannot leave the key un-dropped.
411 on_key_rejected(rejected_key)
413 for chunk_start in range(0, len(all_dois), _MAX_DOIS_PER_REQUEST):
414 chunk = all_dois[chunk_start : chunk_start + _MAX_DOIS_PER_REQUEST]
415 doi_filter = "|".join(chunk)
417 params = {
418 "filter": f"doi:{doi_filter}",
419 "per_page": str(len(chunk)),
420 "select": "doi,primary_location",
421 }
422 # safe_get auto-injects the project User-Agent. We only override
423 # it here when an email is configured to identify this client.
424 headers: Dict[str, str] = {"Accept": "application/json"}
425 if email:
426 headers["User-Agent"] = f"{USER_AGENT} ({email})"
428 # Defaults bind the loop variables (ruff B023): within a chunk the
429 # params, headers and key are fixed.
430 def _send(
431 with_api_key: bool,
432 _params: Dict[str, str] = params,
433 _headers: Dict[str, str] = headers,
434 _api_key: Optional[str] = api_key,
435 ) -> Any:
436 request_headers = dict(_headers)
437 if with_api_key and _api_key:
438 # Bearer header rather than the equally-supported
439 # ?api_key= query parameter, so the key never lands in a
440 # URL. https://help.openalex.org/api/authentication/
441 request_headers["Authorization"] = f"Bearer {_api_key}"
442 return safe_get(
443 f"{_OPENALEX_API}/works",
444 params=_params,
445 headers=request_headers,
446 timeout=OPENALEX_ENRICHMENT_API_TIMEOUT,
447 require_https=True,
448 )
450 try:
451 # A refused key is dropped here for the rest of this
452 # enrichment pass, so the remaining chunks do not each pay a
453 # wasted round-trip with a credential already refused.
454 response, _ = send_with_key_fallback(
455 _send,
456 api_key=api_key,
457 context="DOI enrichment",
458 on_key_rejected=_drop_rejected_key,
459 )
461 # No 401/403 branch of its own: the helper has already logged
462 # the one "key refused" warning if a key was involved, and a
463 # keyless 401/403 is not a key problem at all. Either way the
464 # batch is skipped with exactly the warning this module used
465 # before the key existed.
466 if response.status_code != 200:
467 logger.warning(
468 f"DOI enrichment: OpenAlex returned {response.status_code}"
469 )
470 continue
472 data = response.json()
473 works = data.get("results", [])
475 for work in works:
476 work_doi = work.get("doi", "")
477 if not work_doi: 477 ↛ 478line 477 didn't jump to line 478 because the condition on line 477 was never true
478 continue
480 # Normalize for matching
481 work_doi_normalized = _normalize_doi(work_doi)
483 # Extract source info
484 location = work.get("primary_location") or {}
485 source = location.get("source") or {}
486 source_id_raw = source.get("id", "")
487 source_type = source.get("type")
489 if not source_id_raw:
490 continue
492 # Extract short ID from URL
493 source_id = source_id_raw.split("/")[-1]
495 # Apply to all results with this DOI
496 indices = doi_to_indices.get(work_doi_normalized, [])
497 for idx in indices:
498 results[idx]["openalex_source_id"] = source_id
499 if source_type:
500 results[idx]["source_type"] = source_type
501 enriched_count += 1
503 except Exception as exc:
504 # logger.warning rather than logger.exception: this frame's
505 # locals hold ``headers`` (the "Authorization: Bearer <key>"
506 # value) and ``api_key``, which loguru's diagnose would render
507 # into the traceback. Same documented trade-off as
508 # ``search_engine_nasa_ads``. This module is not a
509 # BaseSearchEngine, so it scrubs the key itself rather than
510 # relying on ``_scrub_error``/``_secret_attrs``.
511 safe_msg = scrub_error(exc, api_key, rejected_key)
512 logger.warning(
513 f"DOI enrichment: OpenAlex batch lookup failed: {safe_msg}"
514 )
515 # Graceful: results pass through unenriched
516 continue
518 if enriched_count > 0:
519 logger.info(
520 f"DOI enrichment: resolved {enriched_count} of "
521 f"{len(doi_to_indices)} DOIs to OpenAlex source IDs"
522 )
523 else:
524 logger.debug(
525 f"DOI enrichment: no matches from {len(doi_to_indices)} DOIs"
526 )
528 return results