Coverage for src/local_deep_research/web_search_engines/engines/search_engine_library.py: 97%

209 statements  

« prev     ^ index     » next       coverage.py v7.16.0, created at 2026-09-06 15:42 +0000

1""" 

2Library RAG Search Engine 

3 

4Provides semantic search over the user's personal research library using RAG. 

5""" 

6 

7import json 

8import os 

9from typing import List, Dict, Any, Optional 

10 

11from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity 

12from ...security.secure_logging import logger 

13from ...constants import ( 

14 DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP, 

15 DEFAULT_LOCAL_SEARCH_CHUNK_SIZE, 

16 DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC, 

17 DEFAULT_LOCAL_SEARCH_INDEX_TYPE, 

18 DEFAULT_LOCAL_SEARCH_MODEL, 

19 DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS, 

20 DEFAULT_LOCAL_SEARCH_PROVIDER, 

21 DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE, 

22 DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS, 

23 DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON, 

24 SNIPPET_LENGTH_LONG, 

25) 

26from ...research_library.services.library_rag_service import LibraryRAGService 

27from ...research_library.services.library_service import LibraryService 

28from ...config.thread_settings import get_setting_from_snapshot 

29from ...utilities.chunk_anchor import ( 

30 build_chunk_anchor_url, 

31 extract_chunk_index, 

32 extract_document_id, 

33 is_safe_document_id, 

34) 

35from ...utilities.llm_utils import get_server_url 

36from ...utilities.type_utils import to_bool 

37from ...database.models.library import Collection, Document 

38from ...research_library.services.pdf_storage_manager import PDFStorageManager 

39from ...database.session_context import get_user_db_session 

40from ...config.paths import get_library_directory 

41 

42 

43class LibraryRAGSearchEngine(BaseSearchEngine): 

44 """ 

45 Search engine that queries the user's research library using RAG/semantic search. 

46 """ 

47 

48 # Mark as local RAG engine for tool selection and safety checks 

49 is_local = True 

50 

51 # Egress (ADR-0007): a local document store — sensitive data, kept on-box. 

52 egress_sensitivity = Sensitivity.SENSITIVE 

53 egress_exposure = Exposure.CONTAINED 

54 

55 def __init__( 

56 self, 

57 llm: Optional[Any] = None, 

58 max_filtered_results: Optional[int] = None, 

59 max_results: int = 10, 

60 settings_snapshot: Optional[Dict[str, Any]] = None, 

61 **kwargs, 

62 ): 

63 """ 

64 Initialize the library RAG search engine. 

65 

66 Args: 

67 llm: Language model for relevance filtering 

68 max_filtered_results: Maximum number of results to keep after filtering 

69 max_results: Maximum number of search results 

70 settings_snapshot: Settings snapshot from thread context 

71 **kwargs: Additional engine-specific parameters 

72 """ 

73 super().__init__( 

74 llm=llm, 

75 max_filtered_results=max_filtered_results, 

76 max_results=max_results, 

77 settings_snapshot=settings_snapshot, 

78 **kwargs, 

79 ) 

80 self.username = ( 

81 settings_snapshot.get("_username") if settings_snapshot else None 

82 ) 

83 # Self-stamp the policy registry name so direct instantiations get 

84 # the runtime egress backstop without caller cooperation (the 

85 # factory stamps registry engines, but routes/services may build 

86 # this engine directly). CollectionSearchEngine overrides this 

87 # with its collection key after calling super().__init__. 

88 self._engine_name = "library" 

89 

90 if not self.username: 

91 logger.warning( 

92 "Library RAG search engine initialized without username" 

93 ) 

94 

95 # Get RAG configuration from settings. NB: keyword arguments — the 

96 # signature is (key, default, username, settings_snapshot); these 

97 # calls used to pass (key, settings_snapshot, default) positionally, 

98 # so the snapshot was never consulted and, outside a thread settings 

99 # context, each setting silently resolved to the snapshot DICT 

100 # itself (the misplaced "default"). 

101 self.embedding_model = get_setting_from_snapshot( 

102 "local_search_embedding_model", 

103 default=DEFAULT_LOCAL_SEARCH_MODEL, 

104 settings_snapshot=settings_snapshot, 

105 ) 

106 self.embedding_provider = get_setting_from_snapshot( 

107 "local_search_embedding_provider", 

108 default=DEFAULT_LOCAL_SEARCH_PROVIDER, 

109 settings_snapshot=settings_snapshot, 

110 ) 

111 self.chunk_size = get_setting_from_snapshot( 

112 "local_search_chunk_size", 

113 default=DEFAULT_LOCAL_SEARCH_CHUNK_SIZE, 

114 settings_snapshot=settings_snapshot, 

115 ) 

116 self.chunk_overlap = get_setting_from_snapshot( 

117 "local_search_chunk_overlap", 

118 default=DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP, 

119 settings_snapshot=settings_snapshot, 

120 ) 

121 self.splitter_type = get_setting_from_snapshot( 

122 "local_search_splitter_type", 

123 default=DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE, 

124 settings_snapshot=settings_snapshot, 

125 ) 

126 raw_text_separators = get_setting_from_snapshot( 

127 "local_search_text_separators", 

128 default=DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON, 

129 settings_snapshot=settings_snapshot, 

130 ) 

131 if isinstance(raw_text_separators, str): 

132 try: 

133 self.text_separators = json.loads(raw_text_separators) 

134 except (json.JSONDecodeError, TypeError): 

135 self.text_separators = DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS 

136 elif isinstance(raw_text_separators, list): 136 ↛ 137line 136 didn't jump to line 137 because the condition on line 136 was never true

137 self.text_separators = raw_text_separators 

138 else: 

139 self.text_separators = DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS 

140 

141 self.distance_metric = get_setting_from_snapshot( 

142 "local_search_distance_metric", 

143 default=DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC, 

144 settings_snapshot=settings_snapshot, 

145 ) 

146 self.normalize_vectors = to_bool( 

147 get_setting_from_snapshot( 

148 "local_search_normalize_vectors", 

149 default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS, 

150 settings_snapshot=settings_snapshot, 

151 ), 

152 default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS, 

153 ) 

154 self.index_type = get_setting_from_snapshot( 

155 "local_search_index_type", 

156 default=DEFAULT_LOCAL_SEARCH_INDEX_TYPE, 

157 settings_snapshot=settings_snapshot, 

158 ) 

159 

160 # Extract server URL from settings snapshot for link generation 

161 self.server_url = get_server_url(settings_snapshot) 

162 

163 # Local RAG retrieval is vector-similarity ranked and can return many 

164 # near-duplicate chunks per query — enough to flood an agent's context 

165 # across research iterations. Apply a RAG-specific cap (default 20), 

166 # independent of the global ``search.max_results`` (which targets web 

167 # engines). This OVERRIDES the inherited max_results for local search. 

168 # Default to the inherited max_results so an ABSENT setting preserves 

169 # the caller's value (programmatic callers, tests); real web/CLI runs 

170 # carry the shipped default of 20 via default_settings.json, so they ARE 

171 # capped. A malformed value also falls back to no change. 

172 rag_max_results = get_setting_from_snapshot( 

173 "search.rag.max_results", 

174 default=self.max_results, 

175 settings_snapshot=settings_snapshot, 

176 ) 

177 try: 

178 rag_max_results = int(rag_max_results) 

179 except (TypeError, ValueError): 

180 rag_max_results = self.max_results 

181 if rag_max_results > 0: 181 ↛ 193line 181 didn't jump to line 193 because the condition on line 181 was always true

182 self.max_results = rag_max_results 

183 

184 # Embeddings already rank by cosine similarity, so the LLM relevance 

185 # filter is OPT-IN here (unlike keyword engines that hardcode 

186 # needs_llm_relevance_filter=True). Default on: prune 

187 # semantically-near-but-irrelevant chunks. The factory only ever SETS 

188 # enable_llm_relevance_filter=True (for should_filter engines) and never 

189 # unsets it — and RAG engines are not should_filter (no 

190 # needs_llm_relevance_filter; dynamic ``collection_<uuid>`` name) — so 

191 # this assignment is authoritative. The base filter is a no-op without 

192 # an llm, so it degrades gracefully. 

193 self.enable_llm_relevance_filter = to_bool( 

194 get_setting_from_snapshot( 

195 "search.rag.enable_relevance_filter", 

196 default=True, 

197 settings_snapshot=settings_snapshot, 

198 ), 

199 default=True, 

200 ) 

201 

202 def search( 

203 self, 

204 query: str, 

205 limit: int = 10, 

206 llm_callback=None, 

207 extra_params: Optional[Dict[str, Any]] = None, 

208 ) -> List[Dict[str, Any]]: 

209 """ 

210 Search the library using semantic search. 

211 

212 Args: 

213 query: Search query 

214 limit: Maximum number of results to return 

215 llm_callback: Optional LLM callback for processing results 

216 extra_params: Additional search parameters 

217 

218 Returns: 

219 List of search results with title, url, snippet, etc. 

220 """ 

221 if not self.username: 

222 logger.error("Cannot search library without username") 

223 return [] 

224 

225 # search() can be called directly (bypassing run()), so apply the 

226 # runtime egress backstop here too. Memoized per snapshot, so the 

227 # run() -> _get_previews() -> search() path re-checks for free. 

228 # Raises PolicyDeniedError on denial — deliberately BEFORE the 

229 # broad try/except below. 

230 self._verify_egress_scope() 

231 

232 try: 

233 # Initialize services 

234 library_service = LibraryService(username=self.username) 

235 

236 # Get all collections for this user 

237 collections = library_service.get_all_collections() 

238 if not collections: 

239 logger.info("No collections found for user") 

240 return [] 

241 

242 # Search across all collections and merge results. Each 

243 # (SearchResult, relevance, metadata) tuple is pre-scored per 

244 # its OWN collection's metric before merging, so a collection 

245 # indexed with cosine/dot_product doesn't get compared on the 

246 # wrong scale against one indexed with l2 (see the 

247 # relevance-formula comment below). 

248 all_results = [] 

249 failed_collections = [] 

250 collection_names = { 

251 c.get("id"): c.get("name") or f"Collection {c.get('id')}" 

252 for c in collections 

253 if c.get("id") 

254 } 

255 for collection in collections: 

256 collection_id = collection.get("id") 

257 if not collection_id: 

258 continue 

259 

260 try: 

261 # Get the Collection record directly for frozen embedding settings 

262 with get_user_db_session(self.username) as session: 

263 collection_record = ( 

264 session.query(Collection) 

265 .filter_by(id=collection_id) 

266 .first() 

267 ) 

268 

269 if ( 

270 not collection_record 

271 or not collection_record.embedding_model 

272 ): 

273 logger.debug( 

274 f"No embedding settings found for collection {collection_id}" 

275 ) 

276 continue 

277 

278 # Get embedding settings directly from Collection 

279 embedding_model = collection_record.embedding_model 

280 provider = collection_record.embedding_model_type 

281 if provider is not None: 

282 embedding_provider = ( 

283 provider.value 

284 if hasattr(provider, "value") 

285 else str(provider) 

286 ) 

287 else: 

288 embedding_provider = self.embedding_provider 

289 coll_chunk_size = getattr( 

290 collection_record, "chunk_size", None 

291 ) 

292 chunk_size = ( 

293 int(coll_chunk_size) 

294 if coll_chunk_size is not None 

295 else int( 

296 self.chunk_size 

297 or DEFAULT_LOCAL_SEARCH_CHUNK_SIZE 

298 ) 

299 ) 

300 coll_chunk_overlap = getattr( 

301 collection_record, "chunk_overlap", None 

302 ) 

303 chunk_overlap = ( 

304 int(coll_chunk_overlap) 

305 if coll_chunk_overlap is not None 

306 else int( 

307 self.chunk_overlap 

308 if self.chunk_overlap is not None 

309 else DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP 

310 ) 

311 ) 

312 splitter_type = ( 

313 getattr(collection_record, "splitter_type", None) 

314 or getattr(self, "splitter_type", None) 

315 or DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE 

316 ) 

317 text_separators = ( 

318 collection_record.text_separators 

319 if getattr( 

320 collection_record, "text_separators", None 

321 ) 

322 is not None 

323 else getattr(self, "text_separators", None) 

324 ) 

325 # Thread the stored normalization flag AND distance 

326 # metric through, exactly like CollectionSearchEngine. 

327 # Without normalize_vectors, LibraryRAGService defaults 

328 # True and would L2-normalize the query against a 

329 # raw-vector collection, flipping the top hit. Without 

330 # distance_metric, every collection is labeled "cosine" 

331 # and l2/dot_product hits get the wrong [0,1] mapping in 

332 # the cross-collection merge below. NULL → prior default. 

333 coll_normalize = getattr( 

334 collection_record, "normalize_vectors", None 

335 ) 

336 normalize_vectors = ( 

337 to_bool( 

338 coll_normalize, 

339 default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS, 

340 ) 

341 if coll_normalize is not None 

342 else getattr( 

343 self, 

344 "normalize_vectors", 

345 DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS, 

346 ) 

347 ) 

348 distance_metric = ( 

349 getattr(collection_record, "distance_metric", None) 

350 or getattr(self, "distance_metric", None) 

351 or DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC 

352 ) 

353 # Thread the stored index_type too (like metric/ 

354 # normalize): a legacy NULL-index_type row that is 

355 # physically HNSW would otherwise fall back to "flat", 

356 # mislabelling the store. NULL → "flat". 

357 index_type = ( 

358 getattr(collection_record, "index_type", None) 

359 or getattr(self, "index_type", None) 

360 or DEFAULT_LOCAL_SEARCH_INDEX_TYPE 

361 ) 

362 

363 # Create RAG service with the collection's embedding settings 

364 with LibraryRAGService( 

365 username=self.username, 

366 embedding_model=embedding_model, 

367 embedding_provider=embedding_provider, 

368 chunk_size=chunk_size, 

369 chunk_overlap=chunk_overlap, 

370 splitter_type=splitter_type, 

371 text_separators=text_separators, 

372 normalize_vectors=normalize_vectors, 

373 distance_metric=distance_metric, 

374 index_type=index_type, 

375 ) as rag_service: 

376 # Get RAG stats to check if there are any indexed documents 

377 stats = rag_service.get_rag_stats(collection_id) 

378 if stats.get("indexed_documents", 0) == 0: 

379 logger.debug( 

380 f"No indexed documents in collection {collection_id}" 

381 ) 

382 continue 

383 

384 # Search this collection's vector index via the 

385 # new int-id-keyed store (chunk text is rehydrated 

386 # from the encrypted DB by id — never read from the 

387 # vector store itself; see the SECURITY INVARIANT 

388 # in vector_stores/base.py). 

389 search_results = rag_service.search( 

390 query, collection_id, limit 

391 ) 

392 

393 # Tag each result with collection ID and pre-score 

394 # relevance per THIS collection's metric (so cosine 

395 # and L2 sort compatibly on merge). 

396 for r in search_results: 

397 # Map r.distance to a [0, 1] relevance score: 

398 # - cosine/dot_product: (d+1)/2 clamped to [0,1] 

399 # - l2 / other: 1/(1+d) in (0, 1] 

400 # Must match faiss_store._build_base_index metric 

401 # selection (cosine/dot_product -> IP, else L2). 

402 relevance = ( 

403 max(0.0, min(1.0, (r.distance + 1.0) / 2.0)) 

404 if r.metric in ("cosine", "dot_product") 

405 else 1.0 / (1.0 + r.distance) 

406 ) 

407 all_results.append((r, relevance, collection_id)) 

408 

409 except Exception as e: 

410 safe_msg = self._scrub_error(e) 

411 logger.exception( 

412 f"Error searching collection {collection_id} ({type(e).__name__}): {safe_msg}" 

413 ) 

414 failed_collections.append(collection_id) 

415 continue 

416 

417 # Fail loudly if *every* collection errored (returning [] 

418 # masks outright service failure as "no results"). Partial 

419 # failures with 0 hits also raise: distinguishing a genuine 

420 # "not found" from "the one collection with answers crashed" 

421 # is impossible without this check. 

422 if failed_collections and not all_results: 

423 failed_names = [ 

424 collection_names.get(cid) or str(cid) 

425 for cid in failed_collections 

426 ] 

427 self._raise_collections_failed(failed_names) 

428 

429 if failed_collections and all_results: 

430 failed_names = [ 

431 collection_names.get(cid) or str(cid) 

432 for cid in failed_collections 

433 ] 

434 logger.warning( 

435 f"Library search returned partial results: {len(failed_collections)} of " 

436 f"{len(collections)} collection(s) failed during search: {failed_names}" 

437 ) 

438 

439 if not all_results: 

440 logger.info("No results found across any collections") 

441 return [] 

442 

443 # Sort by pre-scored relevance (descending) and take top limit 

444 all_results.sort(key=lambda x: x[1], reverse=True) 

445 top_results = all_results[:limit] 

446 

447 # Convert to search result format 

448 results = [] 

449 for r, relevance, coll_id in top_results: 

450 metadata = dict(r.metadata or {}) 

451 metadata["collection_id"] = coll_id 

452 metadata["collection_name"] = ( 

453 collection_names.get(coll_id) or "Unknown" 

454 ) 

455 

456 # Get document ID 

457 doc_id = ( 

458 r.source_id 

459 or metadata.get("source_id") 

460 or metadata.get("document_id") 

461 ) 

462 

463 # Get title 

464 title = ( 

465 r.document_title 

466 or metadata.get("document_title") 

467 or metadata.get("title") 

468 or (f"Document {doc_id}" if doc_id else "Untitled") 

469 ) 

470 

471 # Create snippet from content 

472 snippet = ( 

473 r.text[:SNIPPET_LENGTH_LONG] + "..." 

474 if len(r.text) > SNIPPET_LENGTH_LONG 

475 else r.text 

476 ) 

477 

478 # Generate document URL. Validate chunk_idx and sanitise 

479 # doc_id through the shared helpers so a malformed chunk index 

480 # (UUID, boolean, negative) or path-traversal doc_id cannot leak 

481 # into the citation. 

482 chunk_idx = extract_chunk_index(metadata) 

483 # Authoritative id FIRST. ``extract_document_id`` scans its 

484 # first mapping before any later one, so passing ``metadata`` 

485 # first would let a chunk's denormalised metadata override 

486 # ``r.source_id`` — the DocumentChunk FK that decides which 

487 # document the chunk actually belongs to. Pre-#5381 the 

488 # column always won; a divergent metadata id would otherwise 

489 # point the citation at a different document. 

490 sanitised_doc_id = extract_document_id( 

491 {"source_id": doc_id} if doc_id else None, 

492 metadata, 

493 ) 

494 document_url = self._get_document_url( 

495 sanitised_doc_id, chunk_index=chunk_idx 

496 ) 

497 

498 result = { 

499 "title": title, 

500 "snippet": snippet, 

501 "url": document_url, 

502 "link": document_url, # Add "link" for source extraction 

503 "source": "library", 

504 "source_type": "library", 

505 # Authoritative document id at the TOP level, not only 

506 # nested in ``metadata``: SearchResultsCollector rebuilds 

507 # this citation URL and would otherwise have nothing but 

508 # the denormalised metadata to go on. 

509 "source_id": doc_id, 

510 "relevance_score": float(relevance), 

511 "metadata": metadata, 

512 } 

513 

514 results.append(result) 

515 

516 logger.info( 

517 f"Library RAG search returned {len(results)} results for query: {query}" 

518 ) 

519 return results 

520 

521 except Exception as e: 

522 # Re-raise so run() records the failure in metrics instead of 

523 # treating a failed search as "no matching documents". 

524 safe_msg = self._scrub_error(e) 

525 logger.exception( 

526 f"Error searching library RAG ({type(e).__name__}): {safe_msg}" 

527 ) 

528 raise 

529 

530 @staticmethod 

531 def _raise_collections_failed(failed_collections: List[str]) -> None: 

532 """Signal zero results caused (at least partly) by search failures.""" 

533 raise RuntimeError( 

534 f"Library search returned no results and failed for " 

535 f"{len(failed_collections)} collection(s): {failed_collections}" 

536 ) 

537 

538 def _get_previews( 

539 self, 

540 query: str, 

541 limit: Optional[int] = None, 

542 llm_callback=None, 

543 extra_params: Optional[Dict[str, Any]] = None, 

544 ) -> List[Dict[str, Any]]: 

545 """ 

546 Get preview results for the query. 

547 Delegates to the search method, using the configured max_results 

548 when no explicit limit is provided. 

549 """ 

550 if limit is None: 

551 limit = self.max_results 

552 return self.search(query, limit, llm_callback, extra_params) 

553 

554 def _get_full_content( 

555 self, relevant_items: List[Dict[str, Any]] 

556 ) -> List[Dict[str, Any]]: 

557 """ 

558 Get full content for relevant library documents or chunks. 

559 Retrieves chunk text if a chunk index is targeted, else complete document text. 

560 """ 

561 if not self.username: 

562 logger.error("Cannot retrieve full content without username") 

563 return relevant_items 

564 

565 try: 

566 from ...database.models.library import Document, DocumentChunk 

567 from ...database.session_context import get_user_db_session 

568 

569 # Retrieve content for each document or targeted chunk 

570 for item in relevant_items: 

571 metadata = item.get("metadata") or {} 

572 doc_id = ( 

573 item.get("source_id") 

574 or metadata.get("source_id") 

575 or metadata.get("document_id") 

576 ) 

577 if not doc_id: 

578 continue 

579 

580 chunk_idx = extract_chunk_index(metadata) 

581 

582 # Get document/chunk text from database 

583 with get_user_db_session(self.username) as db_session: 

584 if chunk_idx is not None: 

585 query = db_session.query(DocumentChunk).filter( 

586 DocumentChunk.source_id == doc_id, 

587 DocumentChunk.chunk_index == chunk_idx, 

588 ) 

589 coll_id = metadata.get("collection_id") or item.get( 

590 "collection_id" 

591 ) 

592 if coll_id: 

593 coll_name = ( 

594 coll_id 

595 if str(coll_id).startswith("collection_") 

596 else f"collection_{coll_id}" 

597 ) 

598 query = query.filter( 

599 DocumentChunk.collection_name == coll_name 

600 ) 

601 elif metadata.get("collection_name") and str( 601 ↛ 604line 601 didn't jump to line 604 because the condition on line 601 was never true

602 metadata.get("collection_name") 

603 ).startswith("collection_"): 

604 query = query.filter( 

605 DocumentChunk.collection_name 

606 == metadata.get("collection_name") 

607 ) 

608 

609 chunk = query.order_by( 

610 DocumentChunk.collection_name, DocumentChunk.id 

611 ).first() 

612 

613 if chunk is not None: 

614 text = chunk.chunk_text or "" 

615 else: 

616 text = f"Chunk {chunk_idx} not found for document {doc_id}." 

617 

618 item["content"] = text 

619 item["snippet"] = ( 

620 text[:SNIPPET_LENGTH_LONG] + "..." 

621 if len(text) > SNIPPET_LENGTH_LONG 

622 else text 

623 ) 

624 logger.debug( 

625 f"Retrieved chunk {chunk_idx} content for document {doc_id}" 

626 ) 

627 else: 

628 document = ( 

629 db_session.query(Document) 

630 .filter_by(id=doc_id) 

631 .first() 

632 ) 

633 

634 if document and document.text_content: 634 ↛ 570line 634 didn't jump to line 570

635 # Replace snippet with full content 

636 item["content"] = document.text_content 

637 item["snippet"] = ( 

638 document.text_content[:SNIPPET_LENGTH_LONG] 

639 + "..." 

640 if len(document.text_content) 

641 > SNIPPET_LENGTH_LONG 

642 else document.text_content 

643 ) 

644 logger.debug( 

645 f"Retrieved full content for document {doc_id}" 

646 ) 

647 

648 return relevant_items 

649 

650 except Exception as e: 

651 safe_msg = self._scrub_error(e) 

652 logger.exception( 

653 f"Error retrieving full content from library ({type(e).__name__}): {safe_msg}" 

654 ) 

655 return relevant_items 

656 

657 def _get_document_url( 

658 self, doc_id: Optional[str], chunk_index: Optional[int] = None 

659 ) -> str: 

660 """Get the URL for viewing a document, targeting a specific chunk if provided.""" 

661 if not doc_id: 

662 return "#" 

663 # Validate here rather than trusting the caller. Both in-tree call 

664 # sites pre-sanitise via ``extract_document_id``, but this method 

665 # BUILDS a URL path, so an unvalidated id would be interpolated 

666 # straight into it — a ``../..`` id yields a traversal route on all 

667 # three spellings below, not just the chunk one. 

668 if not is_safe_document_id(str(doc_id)): 

669 return "#" 

670 

671 if chunk_index is not None: 

672 # Route through the shared builder instead of re-deriving the 

673 # rule. The hand-rolled test this replaces accepted any 

674 # non-negative int, so it had no ``MAX_CHUNK_INDEX`` bound — 

675 # a third, weaker copy of a contract ``chunk_anchor`` calls 

676 # itself the single source of truth for. On rejection fall 

677 # through to the plain document route rather than emitting an 

678 # anchor that cannot name a chunk. 

679 anchored = build_chunk_anchor_url("", doc_id, chunk_index) 

680 if anchored is not None: 

681 return anchored 

682 

683 document_url = f"/library/document/{doc_id}" 

684 try: 

685 with get_user_db_session(self.username) as session: 

686 document = session.query(Document).filter_by(id=doc_id).first() 

687 if document: 687 ↛ 724line 687 didn't jump to line 724

688 from pathlib import Path 

689 from ...research_library.utils import apply_user_subdir 

690 

691 library_root = get_setting_from_snapshot( 

692 "research_library.storage_path", 

693 default=str(get_library_directory()), 

694 settings_snapshot=self.settings_snapshot, 

695 ) 

696 base_root = ( 

697 Path(os.path.expandvars(library_root)) 

698 .expanduser() 

699 .resolve() 

700 ) 

701 shared_library = get_setting_from_snapshot( 

702 "research_library.shared_library", 

703 default=False, 

704 settings_snapshot=self.settings_snapshot, 

705 ) 

706 # Per-user root with legacy-shared fallback (issue #5521). 

707 per_user_root = apply_user_subdir( 

708 base_root, self.username, shared_library 

709 ) 

710 if PDFStorageManager.pdf_exists( 

711 per_user_root, 

712 document, 

713 session, 

714 legacy_root=base_root, 

715 ): 

716 document_url = f"/library/document/{doc_id}/pdf" 

717 except Exception as e: 

718 safe_msg = self._scrub_error(e) 

719 logger.warning( 

720 f"Error getting document URL for {doc_id} " 

721 f"({type(e).__name__}): {safe_msg}" 

722 ) 

723 

724 return document_url 

725 

726 def close(self): 

727 """Clean up resources.""" 

728 pass