Coverage for src/local_deep_research/research_library/utils/__init__.py: 95%
233 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Shared utility functions for the Research Library."""
3import ipaddress
4import ntpath
5import os
6import posixpath
7import socket
8import subprocess
9import sys
10from datetime import UTC, datetime
11from pathlib import Path
12from typing import Optional
13from urllib.parse import unquote, urlparse
15from fastapi.responses import JSONResponse
16from loguru import logger
17from sqlalchemy import and_, func, or_
19from ...security import redact_url_for_log
21from ...config.paths import get_library_directory
22from ...constants import FILE_PATH_TEXT_ONLY
23from ...database.models.research import ResearchResource
24from ...database.models.library import (
25 Document,
26 DocumentCollection,
27 DocumentStatus,
28)
29from ...utilities.download_url import (
30 normalize_url_for_hash as normalize_url_for_hash,
31 get_url_hash as get_url_hash,
32 get_legacy_url_hashes as get_legacy_url_hashes,
33 get_url_hash_candidates as get_url_hash_candidates,
34)
35from ...security.ip_ranges import PRIVATE_IP_RANGES
36from ...security.network_utils import is_private_ip
37from ...security.path_validator import PathValidator, resolve_allow_missing
40def escape_like(text: str) -> str:
41 """Escape SQL LIKE/ILIKE wildcards so user input matches literally.
43 ``%`` and ``_`` are LIKE wildcards and ``\\`` is the escape character;
44 without escaping, a query like ``my_note`` would treat ``_`` as "any
45 character" and ``%`` would match anything. Use the result with
46 ``.like(pattern, escape="\\\\")`` / ``.ilike(pattern, escape="\\\\")``.
48 This is the single source of truth for LIKE-escaping across the library
49 and notes query paths (previously copy-pasted in NoteService,
50 LibraryService and unified_search_routes).
51 """
52 return (
53 (text or "")
54 .replace("\\", "\\\\")
55 .replace("%", "\\%")
56 .replace("_", "\\_")
57 )
60def _decode_ipv4_literal(hostname: str) -> Optional[str]:
61 """Decode an ASCII IPv4 literal using the platform's ``inet_aton``.
63 This handles decimal, hexadecimal, octal and shortened dotted forms
64 such as ``2130706433``, ``0x7f000001``, ``0177.0.0.1`` and ``127.1``.
65 It performs no DNS lookup and returns None when parsing fails.
66 Callers normalize IDNA hostnames before invoking this helper.
67 """
68 if not hostname:
69 return None
70 try:
71 packed = socket.inet_aton(hostname)
72 except OSError:
73 return None
74 return socket.inet_ntoa(packed)
77def _is_blocked_private_host(hostname: str) -> bool:
78 """Identify private, local and CGNAT hosts without resolving DNS.
80 IDNA normalization covers Unicode numeric labels and dot separators.
81 Trailing dots and legacy IPv4 literals are handled here without
82 changing the shared ``is_private_ip`` helper — and this normalization
83 (including the trailing-dot strip) applies only to this private-host
84 check; it never feeds the allowlist match in ``is_downloadable_domain``,
85 which stays an exact/dot-suffix comparison against the literal hostname
86 (see ``test_trailing_dot_fqdn_is_refused``). The hostname is also
87 percent-decoded one pass via ``urllib.parse.unquote`` before any of the
88 above, so an encoded numeric IPv4 disguise such as ``%31%32%37.0.0.1``
89 (``127.0.0.1``) is normalized to the same candidate a plain literal
90 would produce, rather than sailing past every subsequent check as an
91 opaque, non-matching hostname string. Each parsed candidate — and, for
92 the IPv4-mapped form, its embedded IPv4 — is checked against the shared
93 ``security.ip_ranges.PRIVATE_IP_RANGES`` recognition table (the same
94 table the fetch chokepoint uses), so every wrapped IPv6 spelling of a
95 private destination that the table knows (6to4, NAT64, IPv4-compatible,
96 IPv4-translated, ...) is rejected here too, not just the CGNAT range
97 behind the ``ipv4_mapped`` unwrap.
98 """
99 normalized = unquote(hostname).rstrip(".")
100 try:
101 normalized = normalized.encode("idna").decode("ascii").rstrip(".")
102 except UnicodeError:
103 pass
104 candidates = [normalized]
105 decoded = _decode_ipv4_literal(candidates[0])
106 if decoded is not None:
107 candidates.append(decoded)
109 for candidate in candidates:
110 if is_private_ip(candidate):
111 return True
112 try:
113 address = ipaddress.ip_address(candidate)
114 except ValueError:
115 continue
116 addresses = [address]
117 if isinstance(address, ipaddress.IPv6Address):
118 mapped = address.ipv4_mapped
119 if mapped is not None:
120 addresses.append(mapped)
121 if any(
122 addr in network
123 for addr in addresses
124 for network in PRIVATE_IP_RANGES
125 ):
126 return True
127 return False
130def is_downloadable_domain(url: str) -> bool:
131 """Identify likely academic resources and direct PDF links.
133 This is a relevance filter. Private IP literals, local names and CGNAT
134 are rejected before the academic-domain, PDF and PubMed-path heuristics.
135 Literal normalization (IDNA, trailing dots, legacy/percent-encoded IPv4,
136 IPv4-mapped IPv6) is best-effort, does not resolve DNS, and applies only
137 to that private-host rejection step — the allowlist match below is an
138 exact/dot-suffix comparison against the literal (non-normalized)
139 hostname, so a public hostname that resolves to a private address can
140 still pass, and a trailing-dot FQDN of an allowlisted domain is still
141 refused (see ``test_trailing_dot_fqdn_is_refused``).
143 Fetches must independently use validated transports such as
144 ``security.safe_requests.SafeSession`` or ``safe_get``. Those transports
145 use ``security.ssrf_validator`` for address and redirect validation;
146 a True result here is never permission to bypass those checks.
147 """
148 try:
149 if not url:
150 return False
152 parsed = urlparse(url.lower())
153 hostname = parsed.hostname or ""
154 path = parsed.path or ""
155 query = parsed.query or ""
157 # Reject recognized private/local literals before any heuristic
158 # can accept them. DNS and transport validation happen at fetch time.
159 if _is_blocked_private_host(hostname):
160 return False
162 # Check for direct PDF files
163 if path.endswith(".pdf") or ".pdf?" in url.lower():
164 return True
166 # List of downloadable academic domains
167 downloadable_domains = [
168 "arxiv.org",
169 "biorxiv.org",
170 "medrxiv.org",
171 "ncbi.nlm.nih.gov",
172 "pubmed.ncbi.nlm.nih.gov",
173 "pubmed.gov",
174 "pubmedcentral.nih.gov",
175 "europepmc.org",
176 "semanticscholar.org",
177 "researchgate.net",
178 "academia.edu",
179 "sciencedirect.com",
180 "springer.com",
181 "nature.com",
182 "wiley.com",
183 "ieee.org",
184 "acm.org",
185 "plos.org",
186 "frontiersin.org",
187 "mdpi.com",
188 "acs.org",
189 "rsc.org",
190 "tandfonline.com",
191 "sagepub.com",
192 "oxford.com",
193 "cambridge.org",
194 "bmj.com",
195 "nejm.org",
196 "thelancet.com",
197 "jamanetwork.com",
198 "annals.org",
199 "ahajournals.org",
200 "cell.com",
201 "science.org",
202 "pnas.org",
203 "elifesciences.org",
204 "embopress.org",
205 "journals.asm.org",
206 "microbiologyresearch.org",
207 "jvi.asm.org",
208 "genome.cshlp.org",
209 "genetics.org",
210 "g3journal.org",
211 "plantphysiol.org",
212 "plantcell.org",
213 "aspb.org",
214 "bioone.org",
215 "company-of-biologists.org",
216 "biologists.org",
217 "jeb.biologists.org",
218 "dmm.biologists.org",
219 "bio.biologists.org",
220 "doi.org",
221 "ssrn.com",
222 "openreview.net",
223 ]
225 # Check if hostname matches any downloadable domain
226 for domain in downloadable_domains:
227 if hostname == domain or hostname.endswith("." + domain):
228 return True
230 # Preserve the host-independent PubMed-path heuristic. Real PubMed
231 # hosts match the allowlist above; hostname substrings do not qualify.
232 if "/pubmed/" in path:
233 return True
235 # Check for PDF in path or query parameters
236 if "/pdf/" in path or "type=pdf" in query or "format=pdf" in query:
237 return True
239 return False
241 except Exception:
242 logger.warning(f"Error parsing URL {redact_url_for_log(url)}")
243 return False
246def is_downloadable_url(url: str) -> bool:
247 """Check if a URL is downloadable (academic domain or direct PDF link).
249 This is the single source of truth for downloadability checks.
250 Combines domain checking with PDF extension/path detection.
252 Args:
253 url: The URL to check
255 Returns:
256 True if the URL is from a downloadable academic domain or is a direct PDF link
257 """
258 return is_downloadable_domain(url)
261def get_document_for_resource(session, resource):
262 """Get Document for a ResearchResource.
264 Checks resource.document_id first (library resources point directly
265 to existing Documents), falls back to Document.resource_id lookup
266 (web downloads create Documents with resource_id set).
267 """
268 if resource.document_id:
269 return (
270 session.query(Document).filter_by(id=resource.document_id).first()
271 )
272 return session.query(Document).filter_by(resource_id=resource.id).first()
275def resource_document_clause():
276 """Join resources to their canonical document, honoring explicit links.
278 Mirrors ``get_document_for_resource``: retained documents can still
279 name a resource as their owner after that resource links elsewhere.
280 """
281 return or_(
282 ResearchResource.document_id == Document.id,
283 and_(
284 ResearchResource.document_id.is_(None),
285 ResearchResource.id == Document.resource_id,
286 ),
287 )
290def document_awaits_pdf(document) -> bool:
291 """Whether a completed research download still needs its PDF.
293 "Sync library" keeps a downloaded document whose PDF file was removed
294 from disk if the document still has its extracted text and its own
295 research resource (``resource_id``). It labels the document a PDF and
296 stores it the way text-only documents are stored (``file_type`` "pdf",
297 ``storage_mode`` "none", ``file_path`` ``FILE_PATH_TEXT_ONLY``), resets
298 the tracker that identifies the document (see
299 ``find_tracker_for_document``) if it names that file or no file, and
300 reports the file missing, so every document it reports that way matches
301 here
302 (see ``LibraryService._keep_text_report_missing_pdf``). Everywhere else
303 a completed document counts as downloaded, so ``download_resource`` and
304 the PDF queueing paths (``queue_research_downloads`` and the "Download
305 All New" route, through ``document_awaits_pdf_clause``) ask this, for
306 the document's own resource and for the resources linked to it by
307 content (``ResearchResource.document_id``), and download the PDF again
308 into the same document. One download refills it, so the queueing paths
309 queue one of those resources (see ``DownloadService.refill_rank``).
310 Once the document holds its PDF again, ``download_resource`` reports
311 the document's own resource as already downloaded, and for a linked
312 one whose URL the egress policy allows, ``DownloadService._download_pdf``
313 reuses that stored PDF instead of fetching it. The
314 Download Manager asks too (``LibraryService``'s
315 ``get_research_list_with_stats``,
316 ``get_download_manager_summary_stats`` and ``get_pdf_previews_batch``):
317 such a document counts as available to download, not as downloaded,
318 and shows no "Downloaded" badge. So do the library routes
319 ``get_research_sources`` and ``check_downloads``, which report it as
320 not downloaded, and ``download_source``, which downloads it again
321 rather than answer "Already downloaded" (see
322 ``document_counts_as_downloaded``). Text extraction does not ask: such
323 a document already holds its text.
325 A document whose resource is gone (``resource_id`` NULL) never matches,
326 so nothing asks for its PDF. The sync stores such a document text-only
327 too, without reporting it, so that a later download of the same PDF
328 bytes for another resource stores the PDF in it (see
329 ``LibraryService._clear_missing_pdf``).
331 This checks ``file_type`` "pdf", not only that storage marker. Zotero
332 imports and uploads write the same marker for PDFs and differ only in
333 having no ``resource_id``. A text extraction (``file_type`` "text") has
334 the same ``file_path`` and differs only in the ``"database"`` default of
335 ``storage_mode``, so a writer that set it to "none", the value for a
336 document without a stored PDF, would make every extracted text ask for
337 a PDF. The sync can set the label itself because what it found missing
338 was a PDF: the file paths of documents and trackers come only from PDF
339 storage (``PDFStorageManager.save_pdf``), and the sync converts a document
340 only for a missing file that its own ``file_path`` names or, for a
341 document without a path of its own, that the tracker identifying it
342 names. A tracker counts for a document only when its content hash or
343 explicit document link identifies the document, and its ``"database"``
344 path names no file and so leads to no conversion. So a document it
345 converts with another label, such as a failed-download placeholder that
346 a PDF download completed before ``_download_pdf`` labelled such
347 documents "pdf", is picked up too.
349 No other writer leaves a research download in that shape: a
350 "none"-mode download keeps ``file_path=None``, "Remove PDF to save
351 space" writes ``blob_deleted``, and a text extraction is ``file_type``
352 "text". A download into the document that stores no PDF either, as in
353 storage mode "none" or for a PDF over the size limit, clears the mark,
354 see ``DownloadService._download_pdf``.
355 """
356 return (
357 document.status == DocumentStatus.COMPLETED
358 and document.file_type == "pdf"
359 and document.storage_mode == "none"
360 and document.file_path == FILE_PATH_TEXT_ONLY
361 and document.resource_id is not None
362 )
365def document_awaits_pdf_clause():
366 """:func:`document_awaits_pdf` as a SQL condition on ``Document``.
368 The nullable columns are compared through ``coalesce`` and ``status``
369 is NOT NULL, so on a ``Document`` row the condition is never NULL and
370 its negation is true for every document the function rejects.
371 """
372 return and_(
373 Document.status == DocumentStatus.COMPLETED,
374 func.coalesce(Document.file_type, "") == "pdf",
375 func.coalesce(Document.storage_mode, "") == "none",
376 func.coalesce(Document.file_path, "") == FILE_PATH_TEXT_ONLY,
377 Document.resource_id.isnot(None),
378 )
381def document_counts_as_downloaded(document) -> bool:
382 """Whether a loaded ``Document`` counts as downloaded.
384 A completed document counts, unless it awaits its PDF (see
385 ``document_awaits_pdf``), which the Download Manager offers for
386 download again. Its counts apply the same rule in SQL
387 (``LibraryService``'s ``_counts_as_downloaded``), and the source
388 previews under each research (``get_pdf_previews_batch``) apply this
389 function. So do the library routes that tell whether a research's
390 source is downloaded (``get_research_sources`` and ``check_downloads``)
391 and ``download_source``, which answers "Already downloaded" only for a
392 document that counts.
394 ``status`` loads as a ``DocumentStatus`` member. ``DocumentStatus`` is
395 a plain ``enum.Enum``, so a member never equals its string value
396 (``DocumentStatus.COMPLETED == "completed"`` is False), and the status
397 is compared with the member.
398 """
399 return document.status == DocumentStatus.COMPLETED and not (
400 document_awaits_pdf(document)
401 )
404# When this server process started. This module is imported while the app
405# starts, before any download can claim a queue row. See
406# download_claim_is_live.
407_PROCESS_STARTED_AT = datetime.now(UTC)
410def download_claim_is_live(claimed_at: Optional[datetime]) -> bool:
411 """Whether a PROCESSING download-queue row is still being downloaded.
413 ``claimed_at`` is the row's ``last_attempt_at``, which the download
414 that claims it sets (``_claim_download_queue_item`` in the library
415 routes). A download finishes or releases the row it claimed before it
416 ends, and it cannot outlive this server process, which serves with one
417 worker (see ``web/app.py``). So a row claimed before this process
418 started was cut off by a restart, as is one claimed before claims
419 recorded their time (``claimed_at`` None): nothing finishes or
420 releases it, so it does not stand for a download in progress. A row
421 whose release failed in this process (which is logged) still counts
422 as in progress until the next restart.
424 Both times are wall-clock readings, so a clock that goes back defeats
425 the rule. If this process started with its clock behind the time an
426 earlier process recorded for a claim, as on a device without a
427 real-time clock that starts with a stale time and has it corrected
428 later, that claim counts as in progress until the next restart, and no
429 other resource of its document refills the document until then. A
430 clock set back while this process runs can make a claim it made count
431 as cut off, so that another resource of the document can be queued
432 while that download runs. The queue row records nothing else that
433 tells the process that claimed it.
434 """
435 if claimed_at is None:
436 return False
437 if claimed_at.tzinfo is None: 437 ↛ 438line 437 didn't jump to line 438 because the condition on line 437 was never true
438 claimed_at = claimed_at.replace(tzinfo=UTC)
439 return claimed_at >= _PROCESS_STARTED_AT
442def find_tracker_by_url(session, url: str):
443 """Find a ``DownloadTracker`` for ``url`` across hash schemes.
445 Tries the canonical B2 hash first, then the pre-B2 legacy spellings
446 (see :func:`get_legacy_url_hashes`). Returns ``None`` when no row
447 matches. Read-compat only — callers creating a tracker must still
448 write the canonical hash.
450 Implemented as sequential ``filter_by`` lookups (not a single
451 ``IN``) so existing ``MagicMock`` tests that force
452 ``filter_by(...).first() -> None`` keep forcing a miss on every
453 candidate.
454 """
455 from ...database.models.download_tracker import DownloadTracker
457 candidates = get_url_hash_candidates(url)
458 for candidate in candidates:
459 tracker = (
460 session.query(DownloadTracker).filter_by(url_hash=candidate).first()
461 )
462 if tracker is not None:
463 return tracker
464 return None
467def find_tracker_for_document(session, document: Document):
468 """Find a URL tracker whose content identity belongs to this document.
470 URL equivalence alone is insufficient for destructive operations: legacy
471 trackers can contain different PDF versions at the same canonical URL.
472 Require a matching content hash or explicit document link, reject
473 contradictory identity fields, and keep looking through legacy hashes
474 when the canonical tracker belongs to another document.
475 """
476 from ...database.models.download_tracker import DownloadTracker
478 if not document.original_url:
479 return None
480 for candidate in get_url_hash_candidates(document.original_url):
481 tracker = (
482 session.query(DownloadTracker).filter_by(url_hash=candidate).first()
483 )
484 if tracker is None:
485 continue
486 if tracker.library_document_id and (
487 tracker.library_document_id != document.id
488 ):
489 continue
490 if tracker.file_hash and tracker.file_hash != document.document_hash:
491 continue
492 if tracker.library_document_id == document.id or (
493 tracker.file_hash and tracker.file_hash == document.document_hash
494 ):
495 return tracker
496 return None
499def ensure_in_collection(
500 session, document_id: str, collection_id: str
501) -> "DocumentCollection":
502 """Get or create a DocumentCollection link between a document and a collection.
504 Args:
505 session: SQLAlchemy session
506 document_id: UUID of the document
507 collection_id: UUID of the collection
509 Returns:
510 The existing or newly created DocumentCollection row
511 """
512 existing = (
513 session.query(DocumentCollection)
514 .filter_by(document_id=document_id, collection_id=collection_id)
515 .first()
516 )
517 if existing:
518 return existing
520 doc_collection = DocumentCollection(
521 document_id=document_id,
522 collection_id=collection_id,
523 indexed=False,
524 )
525 session.add(doc_collection)
526 return doc_collection
529def _reject_unsafe_username_component(username: str) -> None:
530 """Fail closed unless ``username`` is a single, safe path component.
532 ``apply_user_subdir`` joins the username directly into a filesystem path,
533 so any value that could let a downstream ``.resolve()`` escape the
534 library base would be an arbitrary-directory read/write primitive. This
535 mirrors registration's *exact* predicate
536 (``username.replace('_','').replace('-','').isalnum()``) so the two
537 checks can never diverge: any Unicode letters/digits registration accepts
538 (e.g. accented Latin, Cyrillic, CJK) are accepted here too, alongside
539 plain ``[A-Za-z0-9_-]``. ``str.isalnum()`` still rejects every
540 path-escape vector — separators (``/``, ``\\``), ``..``, ``:``, dots,
541 spaces, and control characters are never alphanumeric — so this remains a
542 fail-closed traversal guard; it does not attempt to additionally block
543 reserved device names or homoglyphs, since neither is meaningfully closed
544 by matching registration's charset. We reject rather than sanitize/hash
545 so a valid account keeps a human-readable per-user directory, and reject
546 rather than fall back to the shared base (which would risk cross-user
547 co-mingling).
548 """
549 stripped = username.replace("_", "").replace("-", "")
550 if not stripped or not stripped.isalnum():
551 raise ValueError("Unsafe username for per-user library path")
554def apply_user_subdir(
555 base_path, username: Optional[str], shared_library: bool = False
556) -> Path:
557 """Return the per-user library directory under ``base_path``.
559 Single source of truth for the per-user library layout (issue #5521):
560 downloaded PDFs live under ``<base>/<username>`` so two users' per-user
561 autoincrement ``resource_id`` filenames (e.g. ``pdfs/5.pdf``) can no
562 longer collide in one shared directory. In shared-library mode — or when
563 no username is available — the base directory itself is returned.
565 The ``username`` is validated as a single safe path component before it is
566 joined (see :func:`_reject_unsafe_username_component`); an unsafe value
567 raises ``ValueError`` rather than escaping the base or silently sharing it.
569 Unlike :func:`get_library_storage_path`, this works from an
570 already-resolved base path plus an explicit ``shared_library`` flag, so
571 background threads (the scheduler download path) and snapshot-based
572 callers (collection/library search engines) get the same layout without
573 depending on request-context settings resolution.
574 """
575 # Expand $VARs (parity with the write path in DownloadService.__init__);
576 # the read-side resolvers previously only expanduser()'d, so a storage_path
577 # containing an env-var token resolved to a different, non-existent path on
578 # read than on write (files became unfindable, tracker state corrupted).
579 base = Path(os.path.expandvars(str(base_path))).expanduser().resolve()
580 # ``shared_library`` removes the per-user directory boundary. Its flag is
581 # user-editable, so shared mode only takes effect when an operator has
582 # explicitly enabled it via the environment. The storage path itself is
583 # operator-only; a path saved before that policy remains in place.
584 if (shared_library and _shared_library_allowed()) or not username:
585 return base
586 _reject_unsafe_username_component(username)
587 return base / username
590def _shared_library_allowed() -> bool:
591 """Whether the operator enabled shared-library mode (env-only gate).
593 Environment-only, mirroring ``filesystem_pdf_storage_allowed`` and
594 ``policy.allow_unprotected_egress``: shared mode drops the per-user
595 directory isolation, so it cannot be turned on through the user-writable
596 settings API — only via
597 ``LDR_RESEARCH_LIBRARY_ALLOW_SHARED_LIBRARY=true``.
598 """
599 from ...settings.env_registry import get_env_setting
601 return bool(get_env_setting("research_library.allow_shared_library", False))
604def get_library_storage_path(username: str) -> Path:
605 """
606 Get the storage path for a user's library.
608 Uses the settings system which respects environment variable overrides:
609 - research_library.storage_path: Base path for library storage
610 - research_library.shared_library: If true, all users share the same directory
612 Args:
613 username: The username
615 Returns:
616 Path to the library storage directory
617 """
618 from ...utilities.db_utils import get_settings_manager
620 settings = get_settings_manager()
622 # Get the base path from settings (uses centralized path, respects LDR_DATA_DIR)
623 base_path = settings.get_setting(
624 "research_library.storage_path",
625 str(get_library_directory()),
626 )
628 # Check if shared library mode is enabled
629 shared_library = settings.get_setting(
630 "research_library.shared_library", False
631 )
633 storage_path = apply_user_subdir(base_path, username, shared_library)
635 # Lazy import to avoid import cycles (this module is imported early by
636 # several bootstrap paths).
637 from ...security.directory_creation import create_directory
639 # ``apply_user_subdir`` has already expanded/resolved the base path and
640 # validated the per-user component; route the actual mkdir through the
641 # audited chokepoint, keeping containment to the resolved base so a
642 # symlinked parent that escapes it is still rejected.
643 resolved_base = (
644 Path(os.path.expandvars(str(base_path))).expanduser().resolve()
645 )
646 create_directory(
647 storage_path,
648 context="per-user library storage",
649 root=resolved_base,
650 )
651 return storage_path
654def open_file_location(file_path: str) -> bool:
655 """
656 Open the file location in the system file manager.
658 Args:
659 file_path: Path to the file
661 Returns:
662 True if successful, False otherwise
663 """
664 try:
665 # Validate path is safe (blocks system dirs, path traversal)
666 validated = PathValidator.validate_local_filesystem_path(file_path)
667 folder = str(validated.parent)
668 if sys.platform == "win32": 668 ↛ 669line 668 didn't jump to line 669 because the condition on line 668 was never true
669 os.startfile(folder)
670 elif sys.platform == "darwin": # macOS
671 result = subprocess.run(
672 ["open", folder],
673 capture_output=True,
674 text=True,
675 shell=False,
676 timeout=10,
677 )
678 if result.returncode != 0: 678 ↛ 679line 678 didn't jump to line 679 because the condition on line 678 was never true
679 logger.error(f"Failed to open folder on macOS: {result.stderr}")
680 return False
681 else: # Linux
682 result = subprocess.run(
683 ["xdg-open", folder],
684 capture_output=True,
685 text=True,
686 shell=False,
687 timeout=10,
688 )
689 if result.returncode != 0:
690 logger.error(f"Failed to open folder on Linux: {result.stderr}")
691 return False
692 return True
693 except Exception:
694 logger.exception("Failed to open file location")
695 return False
698def get_absolute_library_path(
699 relative_path: str, username: str
700) -> Optional[Path]:
701 """
702 Get the absolute path from a relative library path.
704 Uses PathValidator to prevent path traversal attacks, with the same
705 trusted-storage-leaf carve-out as every other library resolver (see
706 ``_resolve_within_root``).
708 Args:
709 relative_path: The relative path from library root
710 username: The username
712 Returns:
713 The absolute path, or None if the path is unsafe
714 """
715 library_root = get_library_storage_path(username)
716 return _resolve_within_root(relative_path, library_root)
719# The one sanctioned operator storage layout: the hard-coded "pdfs"
720# leaf under the library root is commonly a symlink onto a larger
721# disk (pinned by test_saves_to_symlinked_pdfs_directory). Containment
722# for stored paths under this leaf is enforced against the *resolved*
723# pdfs directory; every other stored path is confined to the root
724# itself (GHSA-gv87-qcw7-gjr8). This is the single shared
725# implementation of that rule: PDFStorageManager._safe_path_in_root
726# and every settings-based resolver below resolve through it, so the
727# write side and the read/delete side can never drift apart.
728_TRUSTED_STORAGE_LEAF = "pdfs"
730# The path module whose separator stored library paths are written with.
731# PDFStorageManager stores ``str(path.relative_to(library_root))``, so a
732# stored path separates its segments with "/" on POSIX, where a backslash
733# is an ordinary filename character, and with a backslash on Windows.
734_STORED_PATH_MODULE = ntpath if os.name == "nt" else posixpath
737def _containment_base(
738 relative_path: str, library_root: Path
739) -> tuple[Path, str]:
740 """Return ``(base, remainder)`` for checking a stored library path.
742 ``base`` is the directory ``relative_path`` must stay inside and
743 ``remainder`` is the part of it validated against ``base``. A path
744 whose first segment is the trusted ``pdfs`` leaf
745 (``_TRUSTED_STORAGE_LEAF``) is checked against ``library_root / "pdfs"``
746 when that leaf resolves to a directory; containment then runs against
747 the directory the leaf resolves to. The leaf test reads the path as
748 ``validate_safe_path`` does (surrounding whitespace stripped, then
749 ``posixpath.normpath``), so ``pdfs//x.pdf``, `` pdfs/x.pdf`` and
750 ``pdfs/../pdfs/x.pdf`` count as ``pdfs/x.pdf``. On Windows a backslash
751 is first read as "/", because the store writes ``pdfs\\x.pdf`` there,
752 so the remainder returned with the pdfs base uses "/" only, the one
753 separator werkzeug's ``safe_join`` accepts on Windows. Every other
754 path, and any value that is not a string, is checked unchanged against
755 ``library_root`` itself. ``library_root`` must already be a ``Path``.
756 """
757 if isinstance(relative_path, str): 757 ↛ 771line 757 didn't jump to line 771 because the condition on line 757 was always true
758 normalized = relative_path.strip()
759 if _STORED_PATH_MODULE.sep != "/":
760 normalized = normalized.replace(_STORED_PATH_MODULE.sep, "/")
761 normalized = posixpath.normpath(normalized)
762 first, sep, rest = normalized.partition("/")
763 if first == _TRUSTED_STORAGE_LEAF and sep and rest:
764 trusted = library_root / first
765 try:
766 resolved_trusted = resolve_allow_missing(trusted)
767 except OSError:
768 resolved_trusted = None
769 if resolved_trusted is not None and resolved_trusted.is_dir():
770 return trusted, rest
771 return library_root, relative_path
774def _resolve_within_root(
775 relative_path: str, library_root: Path
776) -> Optional[Path]:
777 """Validate ``relative_path`` inside ``library_root`` (traversal + symlink
778 safe). Returns the absolute path, or ``None`` if unsafe.
780 For a stored path whose FIRST segment is the sanctioned ``pdfs``
781 leaf (``_TRUSTED_STORAGE_LEAF``), containment is checked against
782 the RESOLVED pdfs target directory rather than the library root —
783 exactly as PDFStorageManager does on the write side. No other path
784 shape is loosened: symlink escapes anywhere else still fail closed,
785 and a second-level escape planted inside the resolved pdfs target
786 is caught by the same realpath containment. ``_containment_base``
787 decides which directory applies, and states the accepted spellings.
788 """
789 base, remainder = _containment_base(relative_path, Path(library_root))
790 try:
791 # Use PathValidator to safely join and validate the path
792 safe_path = PathValidator.validate_safe_path(remainder, str(base))
793 result = Path(safe_path)
794 # Block symbolic links to prevent symlink-based escapes
795 if result.is_symlink(): 795 ↛ 796line 795 didn't jump to line 796 because the condition on line 795 was never true
796 logger.warning(f"Symlink blocked: {relative_path}")
797 return None
798 return result
799 except ValueError:
800 logger.warning(f"Path traversal blocked: {relative_path}")
801 return None
804def get_absolute_path_from_settings(
805 relative_path: str,
806 username: Optional[str] = None,
807 *,
808 settings_manager=None,
809) -> Optional[Path]:
810 """Resolve a path inside the user's configured library root.
812 ``settings_manager`` supplies an already-scoped manager for background
813 threads; otherwise use the ambient manager. The returned path is checked
814 for traversal and symlink escapes, even when the file is missing.
816 Missing files never fall back to the old shared directory. Legacy file
817 names alone cannot establish ownership; recover those files explicitly
818 after verifying their owner. The retired legacy-read environment flag has
819 no effect. Operator-enabled shared-library mode remains an explicit root
820 selection, not a fallback from a missing per-user file.
822 A nonempty path requires a username for both reads and deletes. An empty
823 path may still return the configured base root without a user context for
824 trusted cleanup callers that need a containment boundary.
826 Returns the absolute path, or None if it is unsafe. Raises ValueError for
827 a nonempty path without a username.
828 """
829 if relative_path and not username:
830 raise ValueError(
831 "Refusing to resolve a library file without a user context"
832 )
834 if settings_manager is None:
835 from ...utilities.db_utils import get_settings_manager
837 settings = get_settings_manager()
838 else:
839 settings = settings_manager
840 base_path = (
841 Path(
842 os.path.expandvars(
843 settings.get_setting(
844 "research_library.storage_path",
845 str(get_library_directory()),
846 )
847 )
848 )
849 .expanduser()
850 .resolve()
851 )
852 shared_library = settings.get_setting(
853 "research_library.shared_library", False
854 )
855 per_user_root = apply_user_subdir(base_path, username, shared_library)
857 if not relative_path:
858 # An empty path asks for the (per-user) library root itself.
859 return per_user_root
861 return _resolve_within_root(relative_path, per_user_root)
864def get_containment_root_from_settings(
865 relative_path: str,
866 username: Optional[str] = None,
867 *,
868 settings_manager=None,
869) -> Optional[Path]:
870 """Return the resolved directory a stored library path is confined to.
872 This is the directory ``get_absolute_path_from_settings`` validates
873 ``relative_path`` against for the same ``username``: ``<root>/pdfs``
874 for a stored ``pdfs/...`` path when that leaf resolves to a directory
875 (including the sanctioned layout that symlinks it onto a larger disk),
876 and ``<root>`` itself for every other path, where ``<root>`` is what
877 ``get_absolute_path_from_settings("", username)`` returns. The bare
878 library root cannot serve as the unlink boundary: in the symlinked
879 layout every PDF's real location is on the other disk, so the re-check
880 would refuse the delete and leave the file orphaned there.
882 The directory is returned fully resolved, as it is at the time of the
883 call. Destructive callers call this BEFORE resolving the file, and pass
884 the result as the ``allowed_root`` of
885 ``CascadeHelper.delete_filesystem_file`` with
886 ``allowed_root_is_resolved=True``. The unlink re-check then compares
887 the file's real location with this pinned boundary instead of resolving
888 the boundary again. Otherwise a ``pdfs`` directory swapped for a symlink
889 between validation and delete would move the boundary along with the
890 file (the #5481 swap).
892 Returns None if the library root lookup returns None, or if the
893 directory cannot be resolved safely.
894 """
895 library_root = get_absolute_path_from_settings(
896 "", username, settings_manager=settings_manager
897 )
898 if library_root is None:
899 return None
900 base = _containment_base(relative_path, library_root)[0]
901 try:
902 return resolve_allow_missing(base)
903 except OSError:
904 return None
907def handle_api_error(operation: str, error: Exception, status_code: int = 500):
908 """
909 Handle API errors consistently - log internally, return generic message to user.
911 This prevents information exposure by logging full error details internally
912 while returning a generic message to the user.
914 Args:
915 operation: Description of the operation that failed (for logging)
916 error: The exception that occurred
917 status_code: HTTP status code to return (default: 500)
919 Returns:
920 Flask JSON response tuple (response, status_code)
921 """
922 # Log the full error internally with stack trace
923 logger.exception(f"Error during {operation}")
925 # Return generic message to user (no internal details exposed)
926 return JSONResponse(
927 {
928 "success": False,
929 "error": "An internal error occurred. Please try again or contact support.",
930 },
931 status_code=status_code,
932 )