Coverage for src/local_deep_research/research_library/zotero/client.py: 98%
369 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Thin client for the Zotero Web API (https://www.zotero.org/support/dev/web_api/v3).
4Uses the project's :class:`SafeSession` so every request is SSRF-validated
5and response sizes are bounded, rather than pulling in ``pyzotero`` (which
6would bypass those protections). A redirect from HTTPS to HTTP is refused,
7including the attachment storage redirect that
8:meth:`ZoteroClient.download_attachment` follows itself. Only the small slice
9of the API needed for import / incremental sync is implemented:
11- list collections (for the UI picker)
12- list item *versions* for a collection or the whole library
13- fetch full item data for a batch of keys
14- fetch an item's child attachments
15- download an attachment's file bytes
17The sync service uses the ``/items/top?format=versions`` endpoint, which
18returns the FULL ``{itemKey: version}`` map of the current top-level items.
19Reconciling that map against its mapping table lets the sync detect additions,
20metadata changes (version bump) and removals (key gone) — and, unlike a
21``?since=`` delta, removals *from the collection* too. Full item data is then
22fetched only for the keys that actually changed. Pure attachment edits that
23don't bump the parent item's version are not detected by this approach —
24acceptable for v1.
25"""
27import re
28import time
29from dataclasses import dataclass, field
30from html import unescape
31from typing import Dict, List, Optional, Tuple
32from urllib.parse import quote
34import requests
35from loguru import logger
37from ...security import SafeSession
40def _html_to_text(html: str) -> str:
41 """Minimal HTML→text for Zotero note bodies (which are stored as HTML)."""
42 if not html:
43 return ""
44 # Turn block-ish tags into newlines, drop the rest, unescape entities.
45 text = re.sub(r"(?i)<(br|/p|/div|/li|/h[1-6])\s*>", "\n", html)
46 text = re.sub(r"<[^>]+>", "", text)
47 text = unescape(text)
48 # Collapse runs of blank lines / trailing spaces.
49 lines = [ln.strip() for ln in text.splitlines()]
50 return "\n".join(ln for ln in lines if ln).strip()
53ZOTERO_API_BASE = "https://api.zotero.org"
54# Zotero 7 desktop's read-only local API (no API key; can serve local files).
55ZOTERO_LOCAL_API_BASE = "http://localhost:23119/api"
56ZOTERO_API_VERSION = "3"
58# Zotero caps multi-key item requests at 50 keys.
59_MAX_KEYS_PER_REQUEST = 50
60# Pagination page size for list endpoints.
61_PAGE_LIMIT = 100
62# Hard cap on paginated requests per call. Bounds memory and guards against a
63# server that returns full pages forever (ignoring `start`) — without it the
64# while-True paginators below could loop unbounded. 10k pages * 100 = 1M items.
65_MAX_PAGES = 10_000
66# Cap retries on transient errors / explicit Backoff|Retry-After headers.
67_MAX_RETRIES = 4
68_MAX_BACKOFF_SECONDS = 60
71# ---------------------------------------------------------------------------
72# Client-safe diagnostics.
73#
74# Each of these is author-written constant text: no upstream response body, no
75# filesystem path, no credential, no caller input. They exist as names rather
76# than inline literals because the HTTP layer forwards a Zotero failure to the
77# browser only if its text is one of them — see
78# ``CLIENT_SAFE_ZOTERO_MESSAGES`` in ``sync_service.py``, which maps each
79# message to ITSELF, so what a client receives is always one of these module
80# constants and never the exception's own string.
81#
82# The consequence is the useful one: every other raise site in this package —
83# including the ones that interpolate an HTTP status, a retry count or an
84# exception class name — is replaced with a generic message automatically,
85# and a future raise site cannot leak upstream detail by being forgotten.
86# ---------------------------------------------------------------------------
87ZOTERO_KEY_REJECTED_MESSAGE = (
88 "Zotero rejected the API key (HTTP 403). Check the key and that it has "
89 "read access to this library."
90)
91ZOTERO_RESOURCE_NOT_FOUND_MESSAGE = (
92 "Zotero resource not found (HTTP 404). Check the library id / collection "
93 "key."
94)
95# The most common Zotero misconfiguration by far, and unguessable from a bare
96# "request failed": the library-ID setting holds a username instead of the
97# numeric ID. Dropping this text costs the user the one hint that resolves it.
98ZOTERO_LIBRARY_ID_NOT_NUMERIC_MESSAGE = (
99 "Zotero rejected the request (HTTP 400). This usually means the library "
100 "ID is not the NUMERIC userID / group ID from zotero.org/settings/keys "
101 "(e.g. a username was entered instead)."
102)
105class ZoteroError(Exception):
106 """Base error for Zotero API interactions."""
109class ZoteroAuthError(ZoteroError):
110 """Raised on authentication / authorization failures (HTTP 403)."""
113class ZoteroTransientError(ZoteroError):
114 """Raised when a request fails for a transient reason (network error, or
115 HTTP 429/5xx) after retries are exhausted.
117 Distinct from a plain :class:`ZoteroError` so per-item callers
118 (``download_attachment`` / ``get_attachment_fulltext``) can tell a genuine
119 "this item has no file / no indexed text" (404 → ``None``) apart from a
120 temporary failure. A transient failure must NOT be recorded as "no PDF",
121 otherwise a single rate-limit blip during a bulk sync would permanently
122 downgrade the item to metadata-only until its Zotero version next changes.
123 """
126@dataclass
127class ZoteroAttachment:
128 """A child attachment of a Zotero item."""
130 key: str
131 version: int
132 content_type: Optional[str]
133 link_mode: Optional[str]
134 filename: Optional[str]
135 title: Optional[str]
137 @property
138 def is_downloadable_pdf(self) -> bool:
139 """True if this is a stored PDF we can fetch via the API.
141 ``linked_file`` attachments point at a file on the user's local disk
142 and are not retrievable through the Web API.
143 """
144 return (self.content_type or "").lower() == "application/pdf" and (
145 self.link_mode or ""
146 ) in ("imported_file", "imported_url")
149@dataclass
150class ZoteroItem:
151 """A top-level Zotero item plus its (lazily attached) children."""
153 key: str
154 version: int
155 item_type: str
156 data: Dict = field(default_factory=dict)
157 attachments: List[ZoteroAttachment] = field(default_factory=list)
159 @property
160 def title(self) -> str:
161 return (self.data.get("title") or "").strip()
163 def best_pdf_attachment(self) -> Optional[ZoteroAttachment]:
164 """Return the first downloadable PDF attachment, if any."""
165 for att in self.attachments:
166 if att.is_downloadable_pdf: 166 ↛ 165line 166 didn't jump to line 165 because the condition on line 166 was always true
167 return att
168 return None
170 @property
171 def is_standalone_pdf_attachment(self) -> bool:
172 """A top-level stored PDF with no bibliographic parent.
174 This is what Zotero creates when a PDF is dropped straight into the
175 library without "Retrieve Metadata" / "Create Parent Item" — very
176 common in real libraries. It carries no metadata beyond a filename,
177 but the PDF itself is fully retrievable, so it is importable as its
178 own document. ``linked_file`` attachments are excluded (the Web API
179 cannot serve them).
180 """
181 return (
182 self.item_type == "attachment"
183 and not self.data.get("parentItem")
184 and (self.data.get("contentType") or "").lower()
185 == "application/pdf"
186 and (self.data.get("linkMode") or "").lower()
187 in ("imported_file", "imported_url")
188 )
190 def as_attachment(self) -> ZoteroAttachment:
191 """View this standalone attachment item as its own attachment."""
192 return ZoteroAttachment(
193 key=self.key,
194 version=self.version,
195 content_type=self.data.get("contentType"),
196 link_mode=self.data.get("linkMode"),
197 filename=self.data.get("filename"),
198 title=self.data.get("title") or self.data.get("filename"),
199 )
202def _sleep_for_backoff(headers, attempt: int) -> None:
203 """Honor Zotero ``Backoff`` / ``Retry-After`` headers (capped)."""
204 raw = headers.get("Backoff") or headers.get("Retry-After")
205 delay: float
206 if raw:
207 try:
208 delay = float(raw)
209 except (TypeError, ValueError):
210 delay = 2.0 * attempt
211 else:
212 delay = min(2.0 * attempt, _MAX_BACKOFF_SECONDS)
213 delay = min(max(delay, 0.0), _MAX_BACKOFF_SECONDS)
214 if delay > 0:
215 logger.debug(f"Zotero: backing off {delay:.1f}s (attempt {attempt})")
216 time.sleep(delay)
219class ZoteroClient:
220 """Minimal Zotero Web API client scoped to one library."""
222 def __init__(
223 self,
224 api_key: str,
225 library_type: str,
226 library_id: str,
227 timeout: int = 30,
228 local: bool = False,
229 ):
230 """Initialize the client.
232 Args:
233 api_key: Zotero API key with read access (not needed when local).
234 library_type: ``"user"`` or ``"group"``.
235 library_id: Numeric user or group id.
236 timeout: Per-request timeout in seconds.
237 local: Talk to the desktop Zotero local API on localhost:23119
238 (no API key; can serve local/linked files) instead of the
239 cloud Web API.
240 """
241 if library_type not in ("user", "group"):
242 raise ValueError(f"Invalid library_type: {library_type!r}")
243 if not api_key and not local:
244 raise ValueError("Zotero API key is required")
245 if not str(library_id).strip():
246 raise ValueError("Zotero library_id is required")
248 self.api_key = api_key
249 self.library_type = library_type
250 self.library_id = str(library_id).strip()
251 self.timeout = timeout
252 self.local = local
253 self._base = ZOTERO_LOCAL_API_BASE if local else ZOTERO_API_BASE
254 self._prefix = (
255 f"/{'users' if library_type == 'user' else 'groups'}"
256 f"/{quote(self.library_id, safe='')}"
257 )
258 # The local API is on the loopback interface; allow it explicitly.
259 # What this session fetches is imported into the research library
260 # (item data, notes, annotations, Zotero's full text), so a redirect
261 # from HTTPS to HTTP is refused (#6847). The local API is plain HTTP
262 # from the start, which the guard leaves alone.
263 self.session = SafeSession(
264 allow_localhost=local, block_https_downgrade=True
265 )
266 headers = {"Zotero-API-Version": ZOTERO_API_VERSION}
267 # The local desktop API needs no key and is reached over plaintext HTTP
268 # on the loopback; never attach the key there (it would put a real
269 # credential on the wire for no benefit).
270 if api_key and not local:
271 headers["Zotero-API-Key"] = self.api_key
272 self.session.headers.update(headers)
274 # -- lifecycle ---------------------------------------------------------
276 def close(self) -> None:
277 if getattr(self, "session", None) is not None: 277 ↛ exitline 277 didn't return from function 'close' because the condition on line 277 was always true
278 try:
279 self.session.close()
280 except Exception:
281 logger.debug("Error closing Zotero session", exc_info=True)
282 finally:
283 self.session = None # type: ignore[assignment]
285 def __enter__(self) -> "ZoteroClient":
286 return self
288 def __exit__(self, exc_type, exc_val, exc_tb) -> bool:
289 self.close()
290 return False
292 # -- low-level request -------------------------------------------------
294 def _request(
295 self,
296 path: str,
297 params: Optional[Dict] = None,
298 *,
299 stream: bool = False,
300 allow_redirects: bool = True,
301 prefixed: bool = True,
302 ):
303 """Perform a GET against the Zotero API with retry/backoff.
305 ``path`` is appended to the library prefix (e.g. ``/collections``)
306 unless ``prefixed=False`` (for non-library endpoints like
307 ``/keys/current`` or ``/users/{id}/groups``).
308 Returns the ``requests.Response``. Raises :class:`ZoteroAuthError`
309 on 403 and :class:`ZoteroError` on other non-2xx responses.
311 When ``allow_redirects=False``, a 3xx response is returned as-is
312 (not raised) so the caller can follow the redirect itself — used by
313 ``download_attachment`` to avoid forwarding the API key header to the
314 storage host.
315 """
316 prefix = self._prefix if prefixed else ""
317 url = f"{self._base}{prefix}{path}"
318 last_exc: Optional[Exception] = None
319 for attempt in range(1, _MAX_RETRIES + 1): 319 ↛ 383line 319 didn't jump to line 383 because the loop on line 319 didn't complete
320 try:
321 response = self.session.get(
322 url,
323 params=params,
324 timeout=self.timeout,
325 stream=stream,
326 allow_redirects=allow_redirects,
327 )
328 except (
329 requests.exceptions.Timeout,
330 requests.exceptions.ConnectionError,
331 ) as exc:
332 # Transient network errors — worth retrying.
333 last_exc = exc
334 logger.warning(
335 f"Zotero request error (attempt {attempt}): "
336 f"{type(exc).__name__}"
337 )
338 if attempt == _MAX_RETRIES:
339 raise ZoteroTransientError(
340 f"Zotero request failed: {type(exc).__name__}"
341 ) from exc
342 time.sleep(min(2.0 * attempt, _MAX_BACKOFF_SECONDS))
343 continue
344 except Exception as exc:
345 # Non-transient (e.g. SafeSession SSRF/oversize ValueError, or
346 # a malformed-URL error) — do NOT retry; fail fast.
347 raise ZoteroError(
348 f"Zotero request failed: {type(exc).__name__}"
349 ) from exc
351 status = response.status_code
352 if status == 403:
353 raise ZoteroAuthError(ZOTERO_KEY_REJECTED_MESSAGE)
354 if status == 404:
355 raise ZoteroError(ZOTERO_RESOURCE_NOT_FOUND_MESSAGE)
356 if status in (429, 500, 502, 503, 504):
357 logger.warning(
358 f"Zotero transient HTTP {status} (attempt {attempt})"
359 )
360 if attempt == _MAX_RETRIES:
361 raise ZoteroTransientError(
362 f"Zotero request failed after {attempt} attempts "
363 f"(HTTP {status})"
364 )
365 _sleep_for_backoff(response.headers, attempt)
366 continue
367 # When the caller manages redirects itself, hand back the 3xx.
368 if not allow_redirects and 300 <= status < 400:
369 return response
370 if status == 400:
371 # The most common cause by far: a username (or other
372 # non-numeric value) in the library-ID setting.
373 raise ZoteroError(ZOTERO_LIBRARY_ID_NOT_NUMERIC_MESSAGE)
374 if not (200 <= status < 300):
375 raise ZoteroError(f"Zotero request failed (HTTP {status})")
377 # Honor a Backoff header even on success (server under load).
378 if response.headers.get("Backoff"):
379 _sleep_for_backoff(response.headers, attempt)
380 return response
382 # Unreachable, but keeps type-checkers happy.
383 raise ZoteroError(
384 f"Zotero request failed: {type(last_exc).__name__ if last_exc else 'unknown'}"
385 )
387 @staticmethod
388 def _library_version(response) -> int:
389 try:
390 return int(response.headers.get("Last-Modified-Version", "0"))
391 except (TypeError, ValueError):
392 return 0
394 # -- public API --------------------------------------------------------
396 def test_connection(self) -> int:
397 """Validate credentials by reading the library version.
399 Returns the current library version. Raises on failure.
400 """
401 response = self._request(
402 "/items/top", params={"format": "versions", "limit": 1}
403 )
404 return self._library_version(response)
406 def get_current_key_info(self) -> Dict:
407 """Return ``/keys/current``: the key's owner ``userID`` + access map.
409 Cloud API only (the local API has no key); returns ``{}`` in local
410 mode or when the response isn't JSON. Used to resolve the numeric
411 userID when the user pasted a username into the library-ID field.
412 """
413 if self.local:
414 return {}
415 response = self._request("/keys/current", prefixed=False)
416 try:
417 return response.json() or {}
418 except ValueError:
419 return {}
421 def get_collections(self) -> List[Dict]:
422 """List all collections in the library (paginated).
424 Returns a list of ``{"key", "name", "parentCollection"}`` dicts.
425 """
426 collections: List[Dict] = []
427 start = 0
428 pages = 0
429 while True:
430 pages += 1
431 if pages > _MAX_PAGES:
432 raise ZoteroError(
433 f"Zotero: pagination exceeded safety cap ({_MAX_PAGES} "
434 "pages) — aborting to avoid a runaway loop"
435 )
436 response = self._request(
437 "/collections",
438 params={
439 "format": "json",
440 "limit": _PAGE_LIMIT,
441 "start": start,
442 },
443 )
444 page = response.json()
445 if not page:
446 break
447 for entry in page:
448 data = entry.get("data", {})
449 collections.append(
450 {
451 "key": entry.get("key"),
452 "name": data.get("name", ""),
453 "parentCollection": data.get("parentCollection", False),
454 }
455 )
456 if len(page) < _PAGE_LIMIT:
457 break
458 start += _PAGE_LIMIT
459 return collections
461 def get_groups(self) -> List[Dict]:
462 """List the groups this API key can access (id + name + type).
464 Resolves the key's owner via ``/keys/current`` then reads
465 ``/users/{userID}/groups`` — both non-library-scoped endpoints. Lets
466 the UI offer a group picker instead of making the user hunt for a
467 numeric group id on zotero.org.
468 """
469 if self.local:
470 # The local API has no key endpoint / auth; group discovery isn't
471 # applicable.
472 return []
473 info = self._request("/keys/current", prefixed=False).json()
474 user_id = info.get("userID") if isinstance(info, dict) else None
475 if not user_id:
476 return []
478 groups: List[Dict] = []
479 start = 0
480 pages = 0
481 while True:
482 pages += 1
483 if pages > _MAX_PAGES:
484 raise ZoteroError(
485 f"Zotero: pagination exceeded safety cap ({_MAX_PAGES} "
486 "pages) — aborting to avoid a runaway loop"
487 )
488 response = self._request(
489 f"/users/{quote(str(user_id), safe='')}/groups",
490 params={
491 "format": "json",
492 "limit": _PAGE_LIMIT,
493 "start": start,
494 },
495 prefixed=False,
496 )
497 page = response.json()
498 if not page:
499 break
500 for entry in page:
501 data = entry.get("data", {})
502 groups.append(
503 {
504 "id": str(entry.get("id") or data.get("id") or ""),
505 "name": data.get("name", ""),
506 "type": data.get("type", ""),
507 }
508 )
509 if len(page) < _PAGE_LIMIT:
510 break
511 start += _PAGE_LIMIT
512 return groups
514 def get_collection_name(self, collection_key: str) -> Optional[str]:
515 """Return a single collection's display name, or None if missing."""
516 try:
517 response = self._request(
518 f"/collections/{quote(collection_key, safe='')}",
519 params={"format": "json"},
520 )
521 except ZoteroError:
522 return None
523 data = response.json().get("data", {})
524 return data.get("name")
526 def get_top_item_versions(
527 self,
528 collection_key: Optional[str] = None, # gitleaks:allow
529 ) -> Tuple[Dict[str, int], int]:
530 """Get ``{itemKey: version}`` for current top-level items.
532 Scoped to ``collection_key`` when given, else the whole library.
533 Also returns the library version (from ``Last-Modified-Version``).
535 This is the FULL current set (not a ``?since=`` delta) on purpose: the
536 sync service reconciles it against its mapping table to detect both
537 deletions AND items removed from the collection (which a library-level
538 ``/deleted`` feed would miss). To make that reconciliation safe, this
539 method raises if the paginated result is INCOMPLETE relative to the
540 ``Total-Results`` header — otherwise a truncated response would look
541 like a batch of deletions to the caller.
542 """
543 if collection_key:
544 path = f"/collections/{quote(collection_key, safe='')}/items/top"
545 else:
546 path = "/items/top"
548 versions: Dict[str, int] = {}
549 library_version = 0
550 start = 0
551 pages = 0
552 expected_total: Optional[int] = None
553 while True:
554 pages += 1
555 if pages > _MAX_PAGES:
556 raise ZoteroError(
557 f"Zotero: pagination exceeded safety cap ({_MAX_PAGES} "
558 "pages) — aborting to avoid a runaway loop"
559 )
560 response = self._request(
561 path,
562 params={
563 "format": "versions",
564 "limit": _PAGE_LIMIT,
565 "start": start,
566 },
567 )
568 library_version = max(
569 library_version, self._library_version(response)
570 )
571 # Total-Results is the authoritative item count; capture it once.
572 if expected_total is None: 572 ↛ 578line 572 didn't jump to line 578 because the condition on line 572 was always true
573 total = response.headers.get("Total-Results")
574 try:
575 expected_total = int(total) if total is not None else None
576 except (TypeError, ValueError):
577 expected_total = None
578 page = response.json() or {}
579 # ``format=versions`` returns a JSON object {key: version}.
580 for key, version in page.items():
581 try:
582 versions[key] = int(version)
583 except (TypeError, ValueError):
584 versions[key] = 0
585 # A short page is the authoritative end-of-data signal for both
586 # list and object (``format=versions``) responses.
587 if len(page) < _PAGE_LIMIT:
588 break
589 # When Total-Results is known, stop one round-trip early. Do NOT
590 # fall back to len(versions) when the header is absent — that makes
591 # the check a tautology and would stop after a single full page.
592 if expected_total is not None and len(versions) >= expected_total:
593 break
594 start += _PAGE_LIMIT
596 # Completeness guard: if the server told us how many items exist and we
597 # ended up with fewer (a truncated / short-circuited page), refuse to
598 # return a partial set — the caller treats "absent" keys as deletions,
599 # so a partial map would silently delete real documents.
600 if expected_total is not None and len(versions) < expected_total:
601 raise ZoteroError(
602 "Zotero returned an incomplete version map "
603 f"({len(versions)}/{expected_total} items); aborting so "
604 "missing items are not mistaken for deletions"
605 )
606 return versions, library_version
608 def get_items(self, item_keys: List[str]) -> List[ZoteroItem]:
609 """Fetch full data for the given top-level item keys (batched)."""
610 items: List[ZoteroItem] = []
611 for batch_start in range(0, len(item_keys), _MAX_KEYS_PER_REQUEST):
612 batch = item_keys[batch_start : batch_start + _MAX_KEYS_PER_REQUEST]
613 if not batch: 613 ↛ 614line 613 didn't jump to line 614 because the condition on line 613 was never true
614 continue
615 response = self._request(
616 "/items",
617 params={
618 "format": "json",
619 "itemKey": ",".join(batch),
620 "limit": _MAX_KEYS_PER_REQUEST,
621 },
622 )
623 for entry in response.json() or []:
624 data = entry.get("data", {})
625 items.append(
626 ZoteroItem(
627 key=entry.get("key"),
628 version=int(entry.get("version", 0) or 0),
629 item_type=data.get("itemType", ""),
630 data=data,
631 )
632 )
633 return items
635 def _get_children_raw(self, item_key: str) -> List[Dict]:
636 """Fetch all raw child entries of an item/attachment (paginated)."""
637 entries: List[Dict] = []
638 start = 0
639 pages = 0
640 while True:
641 pages += 1
642 if pages > _MAX_PAGES:
643 raise ZoteroError(
644 f"Zotero: pagination exceeded safety cap ({_MAX_PAGES} "
645 "pages) — aborting to avoid a runaway loop"
646 )
647 response = self._request(
648 f"/items/{quote(item_key, safe='')}/children",
649 params={
650 "format": "json",
651 "limit": _PAGE_LIMIT,
652 "start": start,
653 },
654 )
655 page = response.json() or []
656 entries.extend(page)
657 if len(page) < _PAGE_LIMIT:
658 break
659 start += _PAGE_LIMIT
660 return entries
662 def get_children(self, item_key: str) -> List[ZoteroAttachment]:
663 """Fetch an item's child attachments."""
664 attachments: List[ZoteroAttachment] = []
665 for entry in self._get_children_raw(item_key):
666 data = entry.get("data", {})
667 if data.get("itemType") != "attachment":
668 continue
669 attachments.append(
670 ZoteroAttachment(
671 key=entry.get("key"),
672 version=int(entry.get("version", 0) or 0),
673 content_type=data.get("contentType"),
674 link_mode=data.get("linkMode"),
675 filename=data.get("filename"),
676 title=data.get("title"),
677 )
678 )
679 return attachments
681 def get_notes(self, item_key: str) -> List[str]:
682 """Return the plain text of an item's child notes (HTML stripped)."""
683 notes: List[str] = []
684 for entry in self._get_children_raw(item_key):
685 data = entry.get("data", {})
686 if data.get("itemType") != "note":
687 continue
688 text = _html_to_text(data.get("note") or "")
689 if text: 689 ↛ 684line 689 didn't jump to line 684 because the condition on line 689 was always true
690 notes.append(text)
691 return notes
693 def get_annotations(self, attachment_key: str) -> List[str]:
694 """Return an attachment's annotations as text (highlight + comment).
696 Image/area annotations carry no text (only position), so they're
697 skipped.
698 """
699 out: List[str] = []
700 for entry in self._get_children_raw(attachment_key):
701 data = entry.get("data", {})
702 if data.get("itemType") != "annotation":
703 continue
704 parts = []
705 highlighted = (data.get("annotationText") or "").strip()
706 comment = (data.get("annotationComment") or "").strip()
707 if highlighted:
708 parts.append(highlighted)
709 if comment:
710 parts.append(f"[note] {comment}")
711 if parts:
712 out.append(" — ".join(parts))
713 return out
715 def download_attachment(self, attachment_key: str) -> Optional[bytes]:
716 """Download an attachment's file bytes, or None on failure.
718 Zotero's ``/items/{key}/file`` endpoint typically answers with a 302
719 to the actual storage location (Zotero's S3, or a WebDAV/external host
720 for self-hosted storage). We must NOT follow that redirect with the
721 authenticated session: ``requests`` only strips the standard
722 ``Authorization`` header on a cross-host redirect, so our custom
723 ``Zotero-API-Key`` header would otherwise be forwarded verbatim to the
724 storage host — a credential leak. Instead we fetch with
725 ``allow_redirects=False`` and follow the ``Location`` ourselves using a
726 fresh, credential-free :class:`SafeSession` (which still SSRF-validates
727 the target).
729 Following the ``Location`` ourselves bypasses the session's HTTPS
730 downgrade guard, so the same rule is applied here: when the API
731 answered over HTTPS (cloud mode), a ``Location`` that is not an
732 absolute ``https://`` URL is refused and the method returns None,
733 rather than fetching the file in cleartext where an on-path attacker
734 could replace the bytes stored in the library (#6847). Any such
735 ``Location`` other than ``http://`` was never fetchable here: it is
736 not resolved against the API URL, so a relative or scheme-less value,
737 like any other scheme, fails the storage session's URL validation.
738 Local mode talks to the desktop API over plain HTTP from the start,
739 so it has no HTTPS to leave.
740 """
741 try:
742 response = self._request(
743 f"/items/{quote(attachment_key, safe='')}/file",
744 allow_redirects=False,
745 )
746 except ZoteroTransientError:
747 # Temporary (429/5xx/network): let the caller skip this item and
748 # retry next sync rather than silently import it metadata-only.
749 raise
750 except ZoteroError:
751 logger.warning(
752 f"Zotero: could not download attachment {attachment_key}"
753 )
754 return None
756 if 300 <= response.status_code < 400:
757 location = (response.headers.get("Location") or "").strip()
758 if not location:
759 logger.warning(
760 f"Zotero: attachment {attachment_key} redirect had no "
761 "Location header"
762 )
763 return None
764 if location.lower().startswith("file:"):
765 # The local API redirects to a file:// path on disk. Only honor
766 # it in local mode (never follow a file:// redirect from the
767 # cloud API).
768 if not self.local:
769 logger.warning(
770 "Zotero: ignoring file:// redirect from the cloud API"
771 )
772 return None
773 return self._read_local_file(location)
774 if self._base.lower().startswith(
775 "https://"
776 ) and not location.lower().startswith("https://"):
777 logger.warning(
778 f"Zotero: refusing attachment {attachment_key} redirect "
779 "that leaves HTTPS"
780 )
781 return None
782 return self._fetch_file_unauthenticated(location)
784 # Some configurations serve the file directly (200, no redirect).
785 content = response.content
786 return content or None
788 @staticmethod
789 def _read_local_file(file_url: str) -> Optional[bytes]:
790 """Read bytes from a ``file://`` URL (local API attachment on disk).
792 Only PDF content is honored: the redirect target comes from whatever
793 answers on the local API port, so restricting reads to files with a
794 PDF magic number keeps this from acting as a generic local-file
795 disclosure primitive (defense in depth — the sync layer discards
796 non-PDF bodies anyway).
797 """
798 from urllib.parse import urlparse
799 from urllib.request import url2pathname
801 try:
802 path = url2pathname(urlparse(file_url).path)
803 with open(path, "rb") as fh:
804 data = fh.read()
805 except OSError:
806 logger.warning("Zotero: could not read local attachment file")
807 return None
808 if not data:
809 return None
810 if b"%PDF" not in data[:1024]:
811 logger.warning("Zotero: local attachment is not a PDF; ignoring it")
812 return None
813 return data
815 def _fetch_file_unauthenticated(self, url: str) -> Optional[bytes]:
816 """Fetch a storage URL with a fresh session that carries no Zotero
817 credentials. SafeSession SSRF-validates the URL (and any further
818 redirects) and refuses a further redirect from HTTPS to HTTP.
820 Raises :class:`ZoteroTransientError` on a transient failure (network
821 error or HTTP 429/5xx from the storage host) so the caller retries next
822 sync rather than importing the item metadata-only; returns None when
823 the session refuses the fetch (URL validation, an HTTPS-to-HTTP
824 redirect) and for a genuine "no file here" (other non-2xx / empty
825 body).
826 """
827 try:
828 with SafeSession(block_https_downgrade=True) as session:
829 response = session.get(url, timeout=self.timeout)
830 except (
831 requests.exceptions.Timeout,
832 requests.exceptions.ConnectionError,
833 ) as exc:
834 raise ZoteroTransientError(
835 f"Zotero storage fetch failed: {type(exc).__name__}"
836 ) from exc
837 except Exception as exc:
838 logger.warning(
839 f"Zotero: storage download failed ({type(exc).__name__})"
840 )
841 return None
842 if response.status_code in (429, 500, 502, 503, 504):
843 raise ZoteroTransientError(
844 f"Zotero storage returned HTTP {response.status_code}"
845 )
846 if not (200 <= response.status_code < 300):
847 logger.warning(
848 f"Zotero: storage download returned HTTP {response.status_code}"
849 )
850 return None
851 content = response.content
852 return content or None
854 def get_attachment_fulltext(self, attachment_key: str) -> Optional[str]:
855 """Return Zotero's already-extracted full text for an attachment.
857 Uses ``GET /items/{key}/fulltext``; returns None when Zotero has no
858 indexed text for it (HTTP 404) or the content is empty. Lets the sync
859 skip downloading + re-parsing the PDF in text-only mode.
860 """
861 try:
862 response = self._request(
863 f"/items/{quote(attachment_key, safe='')}/fulltext"
864 )
865 except ZoteroTransientError:
866 # Temporary failure — don't conflate with "no indexed text"; let
867 # the caller retry next sync.
868 raise
869 except ZoteroError:
870 return None
871 try:
872 data = response.json()
873 except Exception:
874 return None
875 if not isinstance(data, dict):
876 return None
877 content = data.get("content")
878 return content or None