Coverage for src/local_deep_research/web_search_engines/engines/search_engine_openalex.py: 97%
224 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""OpenAlex search engine implementation for academic papers and research."""
3from typing import Any, Dict, List, Optional
5import requests
7from langchain_core.language_models import BaseLLM
9from ...constants import SNIPPET_LENGTH_LONG, USER_AGENT
10from ...security.safe_requests import safe_get
11from ...security.secure_logging import logger
12from ...settings.exceptions import SettingsValueError
13from ...utilities.openalex_enrichment import (
14 OPENALEX_AUTH_FAILURE_STATUSES,
15 bounded_error_body,
16 normalize_openalex_api_key,
17 send_with_key_fallback,
18)
19from ..rate_limiting import RateLimitError
20from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
23class OpenAlexAuthError(Exception):
24 """OpenAlex refused the request as unauthenticated or forbidden.
26 Raised instead of returning ``[]`` so ``BaseSearchEngine.run`` records
27 the search as *failed* with a message, rather than as a successful
28 search that happened to find nothing. A silently empty result set is
29 indistinguishable from "no papers matched" and hides a misconfigured
30 key indefinitely.
32 Raised only when a key was actually sent *and* the automatic keyless
33 retry also failed. When the retry succeeds the user gets their
34 results and only a warning is logged; when no key was configured at
35 all, a 401/403 keeps the pre-existing "log an API error, return []"
36 behaviour, because there is no key to blame.
37 """
40class OpenAlexSearchEngine(BaseSearchEngine):
41 """OpenAlex search engine implementation with natural language query support."""
43 # Restates the base default. ``openalex_api_key`` is bound to the very
44 # same string object as ``api_key``, so it needs no entry of its own;
45 # ``_rejected_api_key`` does, because once OpenAlex refuses the key
46 # ``self.api_key`` becomes None and ``_scrub_error`` would no longer
47 # have the literal to redact. Kept explicit here (rather than
48 # inherited silently) so a future narrowing of this tuple has to
49 # confront the rejected-key slot.
50 _secret_attrs = ("api_key", "_rejected_api_key")
52 # Mark as public search engine
53 is_public = True
54 egress_sensitivity = Sensitivity.NON_SENSITIVE
55 egress_exposure = Exposure.EXPOSING
56 # Scientific/academic search engine
57 is_scientific = True
58 is_lexical = True
59 needs_llm_relevance_filter = True
61 def __init__(
62 self,
63 max_results: int = 25,
64 email: Optional[str] = None,
65 api_key: Optional[str] = None,
66 sort_by: str = "relevance",
67 filter_open_access: bool = False,
68 min_citations: int = 0,
69 from_publication_date: Optional[str] = None,
70 llm: Optional[BaseLLM] = None,
71 max_filtered_results: Optional[int] = None,
72 settings_snapshot: Optional[Dict[str, Any]] = None,
73 **kwargs,
74 ):
75 """
76 Initialize the OpenAlex search engine.
78 Args:
79 max_results: Maximum number of search results
80 email: Optional email included in the User-Agent for identification;
81 the polite pool was deprecated in February 2026
82 api_key: Optional OpenAlex API key for a higher daily request budget
83 sort_by: Sort order ('relevance', 'cited_by_count', 'publication_date')
84 filter_open_access: Only return open access papers
85 min_citations: Minimum citation count filter
86 from_publication_date: Filter papers from this date (YYYY-MM-DD)
87 llm: Language model for relevance filtering
88 max_filtered_results: Maximum number of results to keep after filtering
89 settings_snapshot: Settings snapshot for configuration
90 **kwargs: Additional parameters to pass to parent class
91 """
92 # Journal filter runs before LLM relevance (Tiers 1-3 are instant)
93 preview_filters = []
94 journal_filter = self._create_journal_filter(
95 "openalex", llm, settings_snapshot
96 )
97 if journal_filter is not None:
98 preview_filters.append(journal_filter)
100 super().__init__(
101 llm=llm,
102 max_filtered_results=max_filtered_results,
103 max_results=max_results,
104 preview_filters=preview_filters, # type: ignore[arg-type]
105 settings_snapshot=settings_snapshot,
106 **kwargs,
107 )
109 self.sort_by = sort_by
110 self.filter_open_access = filter_open_access
111 self.min_citations = min_citations
112 # Only set from_publication_date if it's not empty or "False"
113 self.from_publication_date = (
114 from_publication_date
115 if from_publication_date and from_publication_date != "False"
116 else None
117 )
119 # Get email from settings if not provided
120 if not email and settings_snapshot:
121 from ...config.search_config import get_setting_from_snapshot
123 try:
124 # An explicit default is required: with no ``default``
125 # argument at all, a snapshot that lacks the key and no
126 # thread-local settings context makes
127 # ``get_setting_from_snapshot`` raise
128 # ``NoSettingsContextError``. The empty string (rather
129 # than None) is the safer spelling because it is falsy
130 # under every version of that function's
131 # "was a default supplied?" test. (exc_info is not
132 # passed: loguru ignores it, so it never produced a
133 # traceback anyway.)
134 email = get_setting_from_snapshot(
135 "search.engine.web.openalex.email",
136 default="",
137 settings_snapshot=settings_snapshot,
138 )
139 except SettingsValueError:
140 raise
141 except Exception:
142 logger.debug(
143 "Failed to read openalex.email from settings snapshot"
144 )
146 # Handle "False" string for email
147 self.email = email if email and email != "False" else None
149 if not api_key and settings_snapshot:
150 from ...config.search_config import get_setting_from_snapshot
152 try:
153 # default="" for the same reason as the email lookup
154 # above; ``normalize_openalex_api_key`` turns "" into
155 # None, i.e. "search keyless".
156 api_key = get_setting_from_snapshot(
157 "search.engine.web.openalex.api_key",
158 default="",
159 settings_snapshot=settings_snapshot,
160 )
161 except SettingsValueError:
162 raise
163 except Exception:
164 logger.debug(
165 "Failed to read openalex.api_key from settings snapshot"
166 )
168 # Placeholders, sentinels and non-strings all become None, i.e.
169 # "search keyless". Sending them as a Bearer token would earn a
170 # 401/403 and silently turn a working engine into an empty one.
171 self.api_key: Optional[str] = normalize_openalex_api_key(api_key)
172 self.openalex_api_key: Optional[str] = self.api_key
173 # Set only once OpenAlex refuses the key; see ``_secret_attrs``.
174 self._rejected_api_key: Optional[str] = None
176 # API configuration
177 self.api_base = "https://api.openalex.org"
178 ua_email = self.email
179 self.headers = {
180 "User-Agent": f"{USER_AGENT} ({ua_email})"
181 if ua_email
182 else USER_AGENT,
183 "Accept": "application/json",
184 }
185 if self.api_key:
186 # Bearer header rather than the equally-supported ?api_key=
187 # query parameter, so the key never lands in a URL (logs,
188 # proxies, error messages). Both schemes "work identically" —
189 # https://help.openalex.org/api/authentication/ ("Sending your
190 # key").
191 self.headers["Authorization"] = f"Bearer {self.api_key}"
193 if self.api_key:
194 logger.info(
195 "Using OpenAlex with API key (10x keyless daily budget)"
196 )
197 else:
198 logger.info(
199 "Using OpenAlex without an API key (free tier; a free key from "
200 "https://openalex.org/settings/api raises the daily budget 10x — "
201 "the old email 'polite pool' was deprecated in Feb 2026)"
202 )
204 def _request_works(
205 self, params: Dict[str, Any], *, with_api_key: bool
206 ) -> Any:
207 """Call the OpenAlex works endpoint, optionally without the key.
209 Args:
210 params: Query parameters for ``/works``.
211 with_api_key: When False, the ``Authorization`` header is
212 dropped so the request uses the keyless free tier.
213 """
214 headers = dict(self.headers)
215 if not with_api_key:
216 headers.pop("Authorization", None)
217 return safe_get(
218 f"{self.api_base}/works",
219 params=params,
220 headers=headers,
221 timeout=30,
222 )
224 def _note_openalex_key_rejected(
225 self, api_key: Optional[str] = None
226 ) -> None:
227 """Latch the rejection *and* drop this engine's own credential.
229 The base implementation only latches, which is all a non-OpenAlex
230 scientific engine needs: its ``self.api_key`` belongs to another
231 vendor and never reaches ``api.openalex.org``. This engine is the
232 one that also holds an OpenAlex key of its own, in
233 ``self.api_key`` / ``self.openalex_api_key`` and in the cached
234 ``Authorization`` header, so the latch alone would leave the
235 refused credential live for its *searches*.
237 That matters when the DOI enrichment pass is the path that learns
238 the key is bad — a search that returned 200 followed by an
239 enrichment 401, e.g. a key revoked mid-run. Enrichment's
240 ``on_key_rejected`` callback is this method; without the drop
241 below the next search on the same instance would send
242 ``Authorization: Bearer <refused key>`` again.
244 The single ``_rejected_api_key`` slot never has to hold two
245 different literals: ``_resolve_openalex_enrichment_key`` reads
246 ``self.openalex_api_key`` first, so whenever this engine has a
247 key the enrichment pass sends that same string.
249 Idempotent, and safe in either order relative to
250 :meth:`_drop_rejected_api_key`: the base method keeps the stored
251 literal with ``or self._rejected_api_key``, and clearing an
252 already-cleared attribute or popping an absent header is a no-op.
253 The literal is passed up before the attributes are cleared, so it
254 is never lost from ``_scrub_error``'s redaction set.
255 """
256 super()._note_openalex_key_rejected(
257 api_key or getattr(self, "api_key", None)
258 )
259 self.api_key = None
260 self.openalex_api_key = None
261 headers = getattr(self, "headers", None)
262 if isinstance(headers, dict): 262 ↛ exitline 262 didn't return from function '_note_openalex_key_rejected' because the condition on line 262 was always true
263 headers.pop("Authorization", None)
265 def _drop_rejected_api_key(self) -> None:
266 """Forget a key OpenAlex refused, for this engine's lifetime.
268 Three things have to happen together for "no path sends a refused
269 key twice" to hold, and all three are done by this engine's
270 :meth:`_note_openalex_key_rejected` override:
272 * the ``Authorization`` header is cleared, not just the
273 attribute, so every later query on this instance goes out
274 keyless;
275 * ``BaseSearchEngine._note_openalex_key_rejected`` latches the
276 rejection, so the DOI enrichment pass later in the *same*
277 ``run()`` resolves ``None`` instead of re-reading the identical
278 key out of the settings snapshot both it and ``__init__`` read;
279 * the literal is retained in ``_rejected_api_key`` so it stays in
280 ``_scrub_error``'s redaction set.
282 Kept as the search path's named entry point (it is what
283 ``send_with_key_fallback`` is handed as ``on_key_rejected``) so
284 the two directions — search-learned and enrichment-learned — meet
285 in one implementation instead of drifting apart.
286 """
287 self._note_openalex_key_rejected(self.api_key)
289 def _reapply_rate_limit(self) -> None:
290 """Account for the extra keyless retry request.
292 ``apply_rate_limit`` is what paces this engine against
293 ``api.openalex.org``; the retry is a second real request, so it
294 gets its own call rather than riding on the first one's budget.
295 """
296 self._last_wait_time = self.rate_tracker.apply_rate_limit(
297 self.engine_type
298 )
300 def _get_previews(self, query: str) -> List[Dict[str, Any]]:
301 """
302 Get preview information for OpenAlex search results.
304 Args:
305 query: The search query (natural language supported!)
307 Returns:
308 List of preview dictionaries
309 """
310 logger.info("Searching OpenAlex")
312 # Build the search URL with parameters
313 params = {
314 "search": query, # OpenAlex handles natural language beautifully
315 "per_page": min(self.max_results, 200), # OpenAlex allows up to 200
316 "page": 1,
317 # Request specific fields including abstract for snippets
318 "select": "id,display_name,publication_year,publication_date,doi,primary_location,authorships,cited_by_count,open_access,best_oa_location,abstract_inverted_index",
319 }
321 # Add optional filters
322 filters = []
324 if self.filter_open_access:
325 filters.append("is_oa:true")
327 if self.min_citations > 0:
328 filters.append(f"cited_by_count:>{self.min_citations}")
330 if self.from_publication_date and self.from_publication_date != "False":
331 filters.append(
332 f"from_publication_date:{self.from_publication_date}"
333 )
335 if filters:
336 params["filter"] = ",".join(filters)
338 # Add sorting
339 sort_map = {
340 "relevance": "relevance_score:desc",
341 "cited_by_count": "cited_by_count:desc",
342 "publication_date": "publication_date:desc",
343 }
344 params["sort"] = sort_map.get(self.sort_by, "relevance_score:desc")
346 try:
347 # Apply rate limiting before making the request (simple like PubMed)
348 self._last_wait_time = self.rate_tracker.apply_rate_limit(
349 self.engine_type
350 )
351 logger.debug(
352 f"Applied rate limit wait: {self._last_wait_time:.2f}s"
353 )
355 # Make the API request. The shared helper is what guarantees
356 # a refused key costs at most one extra request and never the
357 # results: it retries keylessly once and drops the key for
358 # this engine's lifetime. The same helper backs the DOI
359 # enrichment pass and the research-library downloader.
360 logger.info(
361 f"Making OpenAlex API request with params: {sorted(params.keys())}"
362 )
363 # Read once. ``self.api_key`` is shared mutable state (another
364 # thread on this instance can drop it), and the retry decision
365 # below and the OpenAlexAuthError gate further down must agree
366 # about whether *this* request carried a credential.
367 api_key = self.api_key
368 key_was_sent = api_key is not None
369 response, _ = send_with_key_fallback(
370 lambda with_api_key: self._request_works(
371 params, with_api_key=with_api_key
372 ),
373 api_key=api_key,
374 context="OpenAlex search",
375 on_key_rejected=self._drop_rejected_api_key,
376 before_retry=self._reapply_rate_limit,
377 )
379 logger.info(f"OpenAlex API response status: {response.status_code}")
381 # Log rate limit info if available
382 if "x-ratelimit-remaining" in response.headers:
383 remaining = response.headers.get("x-ratelimit-remaining")
384 limit = response.headers.get("x-ratelimit-limit", "unknown")
385 logger.debug(
386 f"OpenAlex rate limit: {remaining}/{limit} requests remaining"
387 )
389 if response.status_code == 200:
390 data = response.json()
391 results = data.get("results", [])
392 meta = data.get("meta", {})
393 total_count = meta.get("count", 0)
395 logger.info(
396 f"OpenAlex returned {len(results)} results (total available: {total_count:,})"
397 )
399 # Log first result structure for debugging
400 if results:
401 first_result = results[0]
402 logger.debug(
403 f"First result keys: {list(first_result.keys())}"
404 )
405 logger.debug(
406 f"First result has abstract: {'abstract_inverted_index' in first_result}"
407 )
408 if "open_access" in first_result:
409 logger.debug(
410 f"Open access structure: {first_result['open_access']}"
411 )
413 # Format results as previews
414 previews = []
415 for i, work in enumerate(results):
416 logger.debug(
417 f"Formatting work {i + 1}/{len(results)}: {(work.get('display_name') or 'Unknown')[:50]}"
418 )
419 preview = self._format_work_preview(work)
420 if preview:
421 previews.append(preview)
422 logger.debug(
423 f"Preview created with snippet: {preview.get('snippet', '')[:100]}..."
424 )
425 else:
426 logger.warning(f"Failed to format work {i + 1}")
428 logger.info(
429 f"Successfully formatted {len(previews)} previews from {len(results)} results"
430 )
431 return previews
433 if response.status_code == 429:
434 # 429 is also the *daily budget* status, not just a burst
435 # limit: https://help.openalex.org/api/errors/ defines it
436 # as "Rate limit or daily credit budget exceeded", and
437 # https://help.openalex.org/api/authentication/ says "Two
438 # things return 429 Too Many Requests: exceeding your
439 # daily budget, or making more than 100 requests per
440 # second". Hence the message points at the free key,
441 # which raises that budget 10x.
442 logger.warning(
443 "OpenAlex rate limit reached; a free API key from "
444 "https://openalex.org/settings/api raises the daily budget 10x"
445 )
446 raise RateLimitError( # noqa: TRY301 — re-raised for base retry handling
447 "OpenAlex rate limit exceeded",
448 retry_after=response.headers.get("Retry-After"),
449 )
451 self._record_search_failure(requests.HTTPError(response=response))
452 # response.text can echo the request (headers included), so it
453 # gets the full dual-scrub — credential shapes AND this
454 # engine's literal key — before it reaches a log sink.
455 # Scrub FIRST, truncate after: cutting at 200 characters
456 # first can split the key so that neither the literal pass
457 # nor the anchored Bearer/Authorization regexes match the
458 # surviving prefix. Same ordering rule as
459 # ``security/log_sanitizer.sanitize_error_for_client``.
460 # ``bounded_error_body`` bounds the *regex passes* (not the
461 # body materialisation: ``response.text`` has already decoded
462 # all of it) by cutting the body back to the last whitespace
463 # character at or before 8192. Cutting mid-token would be
464 # unsafe even at that offset, because scrubbing shrinks the
465 # text — a token-shaped run collapses to 17 characters — and
466 # would pull the fragment the cut created into the surviving
467 # 200. Because ``normalize_openalex_api_key`` refuses any key
468 # containing whitespace, a key always lies inside one
469 # whitespace-free run, so cutting at a whitespace boundary
470 # drops any run straddling it in its entirety and no partial
471 # credential can survive.
472 safe_detail = self._scrub_error(bounded_error_body(response.text))[
473 :200
474 ]
476 if (
477 response.status_code in OPENALEX_AUTH_FAILURE_STATUSES
478 and key_was_sent
479 ):
480 # Reached only when a key was sent AND the keyless retry
481 # above also failed. A keyless install that gets a 401/403
482 # from a proxy has no key to blame, so it keeps the
483 # pre-existing "API error → []" behaviour below rather
484 # than being told to check a key it never configured.
485 logger.error(
486 "OpenAlex rejected the API key: "
487 f"{response.status_code} - {safe_detail}"
488 )
489 raise OpenAlexAuthError( # noqa: TRY301 — re-raised by except OpenAlexAuthError so run() records the failure
490 f"OpenAlex authentication failed (HTTP {response.status_code})"
491 )
493 logger.error(
494 f"OpenAlex API error: {response.status_code} - {safe_detail}"
495 )
496 return []
498 except (RateLimitError, OpenAlexAuthError):
499 # Re-raise for the base class: rate limits drive its retry
500 # handling, auth failures must be recorded as a failed search
501 # rather than disappearing into an empty result list.
502 raise
503 except Exception as e:
504 self._record_search_failure(e)
505 # logger.warning rather than logger.exception: the traceback
506 # frames hold self.headers (the "Authorization: Bearer <key>"
507 # value) and would render it under loguru diagnose. Same
508 # documented trade-off as search_engine_nasa_ads.
509 safe_msg = self._scrub_error(e)
510 logger.warning(
511 f"Error searching OpenAlex ({type(e).__name__}): {safe_msg}"
512 )
513 return []
515 def _format_work_preview(
516 self, work: Dict[str, Any]
517 ) -> Optional[Dict[str, Any]]:
518 """
519 Format an OpenAlex work as a preview dictionary.
521 Args:
522 work: OpenAlex work object
524 Returns:
525 Formatted preview dictionary or None if formatting fails
526 """
527 try:
528 # Extract basic information
529 # Use `or` instead of dict.get default — OpenAlex routinely
530 # returns these keys with explicit None values, which would
531 # bypass the default and crash on downstream string ops.
532 work_id = work.get("id") or ""
533 title = work.get("display_name") or "No title"
534 logger.debug(f"Formatting work: {title[:50]}")
536 # Build snippet from abstract or first part of title
537 abstract = None
538 if work.get("abstract_inverted_index"):
539 logger.debug(
540 f"Found abstract_inverted_index with {len(work['abstract_inverted_index'])} words"
541 )
542 # Reconstruct abstract from inverted index
543 abstract = self._reconstruct_abstract(
544 work["abstract_inverted_index"]
545 )
546 logger.debug(
547 f"Reconstructed abstract length: {len(abstract) if abstract else 0}"
548 )
549 else:
550 logger.debug("No abstract_inverted_index found")
552 snippet = (
553 abstract[:SNIPPET_LENGTH_LONG]
554 if abstract
555 else f"Academic paper: {title}"
556 )
557 logger.debug(f"Created snippet: {snippet[:100]}...")
559 # Get publication info
560 publication_year = work.get("publication_year", "unknown")
561 publication_date = work.get("publication_date", "unknown")
563 # Get venue/journal info
564 venue = work.get("primary_location", {})
565 journal_name = "unknown"
566 openalex_source_id = None
567 source_type = None
568 issn = None
569 if venue:
570 source = venue.get("source", {})
571 if source:
572 journal_name = source.get("display_name") or "unknown"
573 # Extract source ID for journal quality lookups
574 raw_sid = source.get("id") or ""
575 if raw_sid:
576 openalex_source_id = raw_sid.split("/")[-1]
577 source_type = source.get("type")
578 # Forward the linking ISSN so the reputation filter's
579 # Tier 2/3 lookups can use it instead of falling back
580 # to fuzzy name matching.
581 issn = source.get("issn_l") or None
583 # Get authors
584 authors = []
585 for authorship in work.get("authorships", [])[
586 :5
587 ]: # Limit to 5 authors
588 author = authorship.get("author", {})
589 if author: 589 ↛ 585line 589 didn't jump to line 585 because the condition on line 589 was always true
590 authors.append(author.get("display_name", ""))
592 authors_str = ", ".join(authors)
593 if len(work.get("authorships", [])) > 5:
594 authors_str += " et al."
596 # Extract author affiliations for the institution-tier scoring.
597 # Each entry is a dict with the OpenAlex institution id, ROR id,
598 # and display name — the lookup_institution() helper accepts any
599 # of those three.
600 affiliations: list[dict] = []
601 seen_inst_ids: set[str] = set()
602 for authorship in work.get("authorships", []):
603 for inst in authorship.get("institutions", []) or []:
604 raw_id = inst.get("id") or ""
605 short_id = raw_id.split("/")[-1] if raw_id else ""
606 if short_id and short_id in seen_inst_ids: 606 ↛ 607line 606 didn't jump to line 607 because the condition on line 606 was never true
607 continue
608 if short_id: 608 ↛ 610line 608 didn't jump to line 610 because the condition on line 608 was always true
609 seen_inst_ids.add(short_id)
610 affiliations.append(
611 {
612 "openalex_id": short_id or None,
613 "ror": (inst.get("ror") or "")
614 .rstrip("/")
615 .split("/")[-1]
616 or None,
617 "name": inst.get("display_name"),
618 }
619 )
621 # Get metrics
622 cited_by_count = work.get("cited_by_count", 0)
624 # Get URL - prefer DOI, fallback to OpenAlex URL.
625 # `.get("doi", work_id)` is wrong: when the key exists with value
626 # None (common for non-DOI works) it returns None, not the
627 # default. Use `or` so a None DOI falls through to work_id.
628 url = work.get("doi") or work_id
629 if not url.startswith("http"):
630 if url.startswith("https://doi.org/"): 630 ↛ 631line 630 didn't jump to line 631 because the condition on line 630 was never true
631 pass # Already a full DOI URL
632 elif url.startswith("10."):
633 url = f"https://doi.org/{url}"
634 else:
635 url = work_id # OpenAlex URL
637 # Check if open access
638 open_access_info = work.get("open_access", {})
639 is_oa = (
640 open_access_info.get("is_oa", False)
641 if open_access_info
642 else False
643 )
644 oa_url = None
645 if is_oa:
646 best_location = work.get("best_oa_location", {})
647 if best_location: 647 ↛ 652line 647 didn't jump to line 652 because the condition on line 647 was always true
648 oa_url = best_location.get("pdf_url") or best_location.get(
649 "landing_page_url"
650 )
652 return {
653 "id": work_id,
654 "title": title,
655 "link": url,
656 "snippet": snippet,
657 "authors": authors_str,
658 "year": publication_year,
659 "date": publication_date,
660 # Both fields emit None (not the "unknown" sentinel) when
661 # OpenAlex has no venue for this work. Downstream consumers
662 # (citation normalizer, journal reputation filter) treat
663 # missing venue as "no scoring signal", which is accurate;
664 # the old "unknown" sentinel leaked through the normalizer
665 # as a literal container_title and even matched a real
666 # OpenAlex source named "unknown" (h_index=5, Q1) in the
667 # reference DB.
668 "journal": journal_name if journal_name != "unknown" else None,
669 "journal_ref": journal_name
670 if journal_name != "unknown"
671 else None,
672 "issn": issn,
673 "affiliations": affiliations or None,
674 "openalex_source_id": openalex_source_id,
675 "source_type": source_type,
676 "citations": cited_by_count,
677 "is_open_access": is_oa,
678 "oa_url": oa_url,
679 "abstract": abstract,
680 "type": "academic_paper",
681 }
683 except Exception as e:
684 safe_msg = self._scrub_error(e)
685 logger.exception(
686 f"Error formatting OpenAlex work: {work.get('id', 'unknown')} ({type(e).__name__}): {safe_msg}"
687 )
688 return None
690 def _reconstruct_abstract(
691 self, inverted_index: Dict[str, List[int]]
692 ) -> str:
693 """
694 Reconstruct abstract text from OpenAlex inverted index format.
696 Args:
697 inverted_index: Dictionary mapping words to their positions
699 Returns:
700 Reconstructed abstract text
701 """
702 try:
703 # Create position-word mapping
704 position_word = {}
705 for word, positions in inverted_index.items():
706 for pos in positions:
707 position_word[pos] = word
709 # Sort by position and reconstruct
710 sorted_positions = sorted(position_word.keys())
711 words = [position_word[pos] for pos in sorted_positions]
713 return " ".join(words)
715 except Exception:
716 logger.debug("Could not reconstruct abstract from inverted index")
717 return ""
719 def _get_full_content(
720 self, relevant_items: List[Dict[str, Any]]
721 ) -> List[Dict[str, Any]]:
722 """
723 Get full content for relevant items (OpenAlex provides most content in preview).
725 Args:
726 relevant_items: List of relevant preview dictionaries
728 Returns:
729 List of result dictionaries with full content
730 """
731 # OpenAlex returns comprehensive data in the initial search,
732 # so we don't need a separate full content fetch
733 results = []
734 for item in relevant_items:
735 result = {
736 "title": item.get("title", ""),
737 "link": item.get("link", ""),
738 "snippet": item.get("snippet", ""),
739 "content": item.get("abstract", item.get("snippet", "")),
740 # Forward journal quality fields for content filters
741 "journal_ref": item.get("journal_ref"),
742 "openalex_source_id": item.get("openalex_source_id"),
743 "source_type": item.get("source_type"),
744 "affiliations": item.get("affiliations"),
745 "metadata": {
746 "authors": item.get("authors", ""),
747 "year": item.get("year", ""),
748 "journal": item.get("journal", ""),
749 "citations": item.get("citations", 0),
750 "is_open_access": item.get("is_open_access", False),
751 "oa_url": item.get("oa_url"),
752 "affiliations": item.get("affiliations"),
753 },
754 }
755 results.append(result)
757 return results