Coverage for src/local_deep_research/research_library/deletion/services/document_deletion.py: 98%
162 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-20 01:24 +0000
1"""
2Document deletion service.
4Handles:
5- Full document deletion with proper cascade cleanup
6- Blob-only deletion (remove PDF, keep text)
7- Remove from collection (unlink or delete if orphaned)
8"""
10from typing import Dict, List, Any
12from loguru import logger
14from ....constants import FILE_PATH_BLOB_DELETED
15from ....database.models.library import (
16 Collection,
17 Document,
18 DocumentChunk,
19 DocumentCollection,
20 RagDocumentStatus,
21 SourceType,
22)
23from ....database.session_context import get_user_db_session
24from ....database.session_passwords import capture_request_db_password
25from ..utils.cascade_helper import CascadeHelper
28def _is_note_document(session, document) -> bool:
29 """Return True if the document represents a note (source_type='note').
31 Notes have their own deletion API at /api/notes/<id>. Letting them be
32 deleted via the generic document-deletion endpoints would bypass the
33 B1 protection that was specifically built to prevent note data loss
34 (see PROTECTED_COLLECTION_TYPES in collection_deletion.py).
36 The check is local-by-design — keep this module free of imports from
37 the notes package to avoid an import cycle.
38 """
39 note_source = session.query(SourceType).filter_by(name="note").first()
40 return bool(note_source) and document.source_type_id == note_source.id
43class DocumentDeletionService:
44 """Service for document deletion operations."""
46 def __init__(self, username: str):
47 """
48 Initialize document deletion service.
50 Args:
51 username: Username for database session
52 """
53 self.username = username
55 def delete_document(self, document_id: str) -> Dict[str, Any]:
56 """
57 Delete a document and ALL related data.
59 This method ensures complete cleanup:
60 - DocumentChunks (no FK constraint, manual cleanup required)
61 - DocumentBlob (CASCADE handles, but we track for stats)
62 - Filesystem files
63 - FAISS index entries
64 - DownloadTracker update
65 - DocumentCollection links (CASCADE)
66 - RagDocumentStatus (CASCADE)
68 Args:
69 document_id: ID of the document to delete
71 Returns:
72 Dict with deletion details:
73 {
74 "deleted": True/False,
75 "document_id": str,
76 "title": str,
77 "blob_deleted": bool,
78 "blob_size": int,
79 "chunks_deleted": int,
80 "collections_unlinked": int,
81 "error": str (if failed)
82 }
83 """
84 result: Dict[str, Any] = {
85 "deleted": False,
86 "document_id": document_id,
87 "title": None,
88 "blob_deleted": False,
89 "blob_size": 0,
90 "chunks_deleted": 0,
91 "collections_unlinked": 0,
92 "file_deleted": False,
93 }
94 rag_collection_ids: List[str] = []
96 with get_user_db_session(self.username) as session:
97 try:
98 # Get document
99 document = session.get(Document, document_id)
100 if not document:
101 return {
102 "deleted": False,
103 "document_id": document_id,
104 "error": "Document not found",
105 }
107 # Notes are not deletable via the generic document API.
108 # The Notes-collection deletion guard only blocks the
109 # cascade path; without this check a user could wipe an
110 # individual note by passing its UUID here, and the bulk
111 # endpoint amplifies the same gap. Route the caller to
112 # DELETE /api/notes/<id> which goes through NoteService
113 # with the proper version/link cleanup.
114 if _is_note_document(session, document):
115 return {
116 "deleted": False,
117 "document_id": document_id,
118 "title": document.title or "Untitled",
119 "error": (
120 "Notes cannot be deleted via the document API. "
121 "Use DELETE /api/notes/<id> instead."
122 ),
123 "is_note": True,
124 }
126 title = document.title or document.filename or "Untitled"
127 result["title"] = title
129 # 1. Get collections before deletion for chunk cleanup
130 collections = CascadeHelper.get_document_collections(
131 session, document_id
132 )
133 result["collections_unlinked"] = len(collections)
135 # Every collection that may hold FAISS vectors / chunk rows
136 # for this document: the UNION of its linked collections and
137 # the collections that actually have chunk rows. The chunk-row
138 # half matters because a partial/failed reindex can leave
139 # chunks in a collection whose join row is gone or was never
140 # marked indexed. Used by the post-commit RAG purge below.
141 chunk_collection_rows = (
142 session.query(DocumentChunk.collection_name)
143 .filter_by(source_type="document", source_id=document_id)
144 .distinct()
145 .all()
146 )
147 rag_collection_ids = list(
148 {str(c) for c in collections}
149 | {
150 row[0].removeprefix("collection_")
151 for row in chunk_collection_rows
152 if row[0]
153 }
154 )
156 # 2. Count chunks for stats. The rows themselves (and their
157 # FAISS vectors) are removed in the post-commit RAG cleanup
158 # phase below — purge_document_vectors resolves the FAISS
159 # int ids from these rows, so they must outlive this
160 # transaction (mirrors NoteService.delete_note).
161 result["chunks_deleted"] = (
162 session.query(DocumentChunk)
163 .filter(
164 DocumentChunk.source_id == document_id,
165 DocumentChunk.source_type == "document",
166 )
167 .count()
168 )
170 # 3. Get blob size before deletion (for stats)
171 result["blob_size"] = CascadeHelper.get_document_blob_size(
172 session, document_id
173 )
174 result["blob_deleted"] = result["blob_size"] > 0
176 # 4. Delete filesystem file if exists
177 if document.storage_mode == "filesystem" and document.file_path:
178 from ...utils import get_absolute_path_from_settings
180 try:
181 file_path = get_absolute_path_from_settings(
182 document.file_path
183 )
184 if file_path:
185 result["file_deleted"] = (
186 CascadeHelper.delete_filesystem_file(
187 str(file_path)
188 )
189 )
190 except Exception:
191 logger.exception("Failed to delete filesystem file")
193 # 5. Update DownloadTracker
194 CascadeHelper.update_download_tracker(session, document)
196 # 6. Delete the document and all related records (leaves the
197 # DocumentChunk rows for the post-commit cleanup phase).
198 CascadeHelper.delete_document_completely(session, document_id)
199 session.commit()
201 result["deleted"] = True
202 logger.info(
203 f"Deleted document {document_id[:8]}... ({title}): "
204 f"{result['chunks_deleted']} chunks, "
205 f"{result['blob_size']} bytes blob"
206 )
208 except Exception:
209 logger.exception(f"Failed to delete document {document_id}")
210 session.rollback()
211 return {
212 "deleted": False,
213 "document_id": document_id,
214 "error": "Failed to delete document",
215 }
217 # Post-commit FAISS + chunk-row cleanup. Runs after the deletion
218 # transaction has committed — releasing SQLite's single write lock —
219 # so the slow vector-store IO never blocks another writer.
220 if result["deleted"]: 220 ↛ 225line 220 didn't jump to line 225 because the condition on line 220 was always true
221 self._purge_document_rag(
222 document_id, rag_collection_ids, full_delete=True
223 )
225 return result
227 def _purge_document_rag(
228 self,
229 document_id: str,
230 collection_ids: List[str],
231 *,
232 full_delete: bool,
233 ) -> None:
234 """Remove a document's FAISS vectors and chunk rows post-commit.
236 Called after the deletion/unlink transaction has committed —
237 releasing SQLite's single write lock — so the slow vector-store IO
238 here never blocks another writer. For each collection the vectors
239 are purged BEFORE their chunk rows (``purge_document_vectors``
240 resolves the FAISS int ids from those rows), then the rows are
241 deleted.
243 A final sweep then guarantees no chunk row survives even when the
244 per-collection purge could not run — e.g. the DB password is
245 unavailable on an encrypted-DB install, or a collection has no
246 current FAISS index. ``full_delete`` selects the sweep scope: True
247 (the document is gone) sweeps every remaining chunk row for the
248 document; False (a single unlink) sweeps only the listed
249 collections, leaving the document's chunks in its other collections
250 intact.
252 Best-effort: the DB delete already committed, so failures here are
253 logged, not surfaced.
254 """
255 if collection_ids:
256 dbpw = capture_request_db_password(self.username)
257 from ...services.rag_service_factory import get_rag_service
259 for collection_id in collection_ids:
260 try:
261 # A fresh service per collection so the factory resolves
262 # THAT collection's stored embedding model/provider — a
263 # shared instance would compute the wrong index hash and
264 # purge nothing for non-default-embedding collections.
265 with get_rag_service(
266 self.username,
267 collection_id=collection_id,
268 db_password=dbpw,
269 ) as rag:
270 rag.purge_document_vectors(document_id, collection_id)
271 rag.purge_document_chunks(document_id, collection_id)
272 except Exception:
273 # exception=True keeps the stack trace. dbpw is a live
274 # local here, but frame-local values only render under
275 # the opt-in LDR_LOGURU_DIAGNOSE dev flag — every
276 # persistent sink sets diagnose=False — so the password
277 # cannot reach a log in normal operation.
278 logger.opt(exception=True).error(
279 "Failed to remove RAG entries for document {} "
280 "from collection {}",
281 document_id,
282 collection_id,
283 )
285 # Guaranteed backstop: remove any chunk rows the RAG purge could not
286 # reach. Their vectors can't be resolved without the rows, but the
287 # collection-scoped rehydration filter already stops an orphaned
288 # vector from ever surfacing another document's text, so dropping the
289 # rows here keeps the deletion complete and safe.
290 try:
291 with get_user_db_session(self.username) as session:
292 if full_delete:
293 CascadeHelper.delete_document_chunks(session, document_id)
294 else:
295 for collection_id in collection_ids:
296 CascadeHelper.delete_document_chunks(
297 session,
298 document_id,
299 f"collection_{collection_id}",
300 )
301 session.commit()
302 except Exception:
303 logger.opt(exception=True).error(
304 "Failed to sweep residual chunk rows for document {}",
305 document_id,
306 )
308 def delete_blob_only(self, document_id: str) -> Dict[str, Any]:
309 """
310 Delete PDF binary but keep document metadata and text content.
312 This saves database space while preserving searchability.
314 Args:
315 document_id: ID of the document
317 Returns:
318 Dict with deletion details:
319 {
320 "deleted": True/False,
321 "document_id": str,
322 "bytes_freed": int,
323 "storage_mode_updated": bool,
324 "error": str (if failed)
325 }
326 """
327 with get_user_db_session(self.username) as session:
328 try:
329 # Get document
330 document = session.get(Document, document_id)
331 if not document:
332 return {
333 "deleted": False,
334 "document_id": document_id,
335 "bytes_freed": 0,
336 "error": "Document not found",
337 }
339 # Notes are not mutable via the generic blob-delete API.
340 # Without this guard, a note's storage_mode/file_path can
341 # be overwritten with the PDF-blob-deletion sentinel via
342 # either DELETE /library/api/document/<id>/blob OR the
343 # bulk DELETE /library/api/documents/blobs path (which
344 # loops over this method per-id).
345 if _is_note_document(session, document):
346 return {
347 "deleted": False,
348 "document_id": document_id,
349 "bytes_freed": 0,
350 "error": (
351 "Notes cannot be modified via the document API. "
352 "Use the notes API instead."
353 ),
354 "is_note": True,
355 }
357 result = {
358 "deleted": False,
359 "document_id": document_id,
360 "bytes_freed": 0,
361 "storage_mode_updated": False,
362 }
364 # Handle based on storage mode
365 if document.storage_mode == "database":
366 # Delete blob from database
367 result["bytes_freed"] = CascadeHelper.delete_document_blob(
368 session, document_id
369 )
371 elif document.storage_mode == "filesystem":
372 # Delete filesystem file
373 from ...utils import get_absolute_path_from_settings
375 if document.file_path:
376 try:
377 file_path = get_absolute_path_from_settings(
378 document.file_path
379 )
380 if file_path and file_path.is_file():
381 result["bytes_freed"] = file_path.stat().st_size
382 CascadeHelper.delete_filesystem_file(
383 str(file_path)
384 )
385 except Exception:
386 logger.exception("Failed to delete filesystem file")
388 else:
389 # No blob to delete
390 return {
391 "deleted": False,
392 "document_id": document_id,
393 "bytes_freed": 0,
394 "error": "Document has no stored PDF (storage_mode is 'none')",
395 }
397 # Update document to indicate blob is deleted
398 document.storage_mode = "none"
399 document.file_path = FILE_PATH_BLOB_DELETED
400 result["storage_mode_updated"] = True
402 session.commit()
403 result["deleted"] = True
405 logger.info(
406 f"Deleted blob for document {document_id[:8]}...: "
407 f"{result['bytes_freed']} bytes freed"
408 )
410 return result
412 except Exception:
413 logger.exception(
414 f"Failed to delete blob for document {document_id}"
415 )
416 session.rollback()
417 return {
418 "deleted": False,
419 "document_id": document_id,
420 "bytes_freed": 0,
421 "error": "Failed to delete document blob",
422 }
424 def remove_from_collection(
425 self,
426 document_id: str,
427 collection_id: str,
428 ) -> Dict[str, Any]:
429 """
430 Remove document from a collection.
432 If the document is not in any other collection after removal,
433 it will be completely deleted.
435 Args:
436 document_id: ID of the document
437 collection_id: ID of the collection
439 Returns:
440 Dict with operation details:
441 {
442 "unlinked": True/False,
443 "document_deleted": bool,
444 "document_id": str,
445 "collection_id": str,
446 "chunks_deleted": int,
447 "error": str (if failed)
448 }
449 """
450 with get_user_db_session(self.username) as session:
451 try:
452 # Verify document exists
453 document = session.get(Document, document_id)
454 if not document:
455 return {
456 "unlinked": False,
457 "document_deleted": False,
458 "document_id": document_id,
459 "collection_id": collection_id,
460 "error": "Document not found",
461 }
463 # Verify collection exists and document is in it
464 doc_collection = (
465 session.query(DocumentCollection)
466 .filter_by(
467 document_id=document_id, collection_id=collection_id
468 )
469 .first()
470 )
472 if not doc_collection:
473 return {
474 "unlinked": False,
475 "document_deleted": False,
476 "document_id": document_id,
477 "collection_id": collection_id,
478 "error": "Document not in this collection",
479 }
481 # Permanent-home guard: the Notes collection is every
482 # note's home (_get_or_create_notes_collection); nothing
483 # re-links a note after an unlink, so removal here would
484 # silently drop the note from semantic search, the
485 # collection view, and the auto-reindex worker — while
486 # list_notes still shows it. NoteService.remove_from_collection
487 # enforces this for the notes API; this mirrors it for the
488 # generic collection-document API (single + bulk).
489 collection = session.get(Collection, collection_id)
490 if (
491 collection is not None
492 and collection.collection_type == "notes"
493 and _is_note_document(session, document)
494 ):
495 return {
496 "unlinked": False,
497 "document_deleted": False,
498 "document_id": document_id,
499 "collection_id": collection_id,
500 "protected": True,
501 "error": (
502 "Cannot remove a note from its notes "
503 "collection — it is the note's permanent "
504 "home. Use DELETE /api/notes/<id> to delete "
505 "the note itself."
506 ),
507 }
509 result = {
510 "unlinked": False,
511 "document_deleted": False,
512 "document_id": document_id,
513 "collection_id": collection_id,
514 "chunks_deleted": 0,
515 }
517 # Count chunks for this document in this collection (for
518 # stats). Their rows and FAISS vectors are removed in the
519 # post-commit RAG cleanup phase — purge_document_vectors
520 # resolves the FAISS int ids from these rows, so they must
521 # outlive this transaction.
522 collection_name = f"collection_{collection_id}"
523 result["chunks_deleted"] = (
524 session.query(DocumentChunk)
525 .filter(
526 DocumentChunk.source_id == document_id,
527 DocumentChunk.source_type == "document",
528 DocumentChunk.collection_name == collection_name,
529 )
530 .count()
531 )
533 # Remove the link
534 session.delete(doc_collection)
535 # Clear the RagDocumentStatus row for this (document, collection)
536 # — the "indexed" marker read by get_rag_stats / the RAG status
537 # route. Its FK cascades fire only when the Document, Collection,
538 # or RAGIndex is deleted; a mere unlink deletes none of those, so
539 # a leftover row reports the just-removed document as still
540 # indexed in this collection.
541 session.query(RagDocumentStatus).filter_by(
542 document_id=document_id, collection_id=collection_id
543 ).delete(synchronize_session=False)
544 session.flush()
546 # Check if document is in any other collection
547 remaining_count = CascadeHelper.count_document_in_collections(
548 session, document_id
549 )
551 # Default (unlink, and note-orphan): purge only THIS
552 # collection's vectors/chunks; the document (or note) and its
553 # chunks in other collections stay intact.
554 full_delete = False
555 rag_collection_ids: List[str] = [collection_id]
557 if remaining_count == 0 and _is_note_document( 557 ↛ 567line 557 didn't jump to line 567 because the condition on line 557 was never true
558 session, document
559 ):
560 # Note that was orphaned by this removal: skip the
561 # destructive delete_document_completely path. Notes
562 # live behind their own deletion API (DELETE /api/notes/<id>)
563 # and are still discoverable via list_notes() which
564 # filters by source_type_id, not collection membership.
565 # The Notes collection itself acts as the persistent
566 # home for every note via _get_or_create_notes_collection.
567 logger.info(
568 f"Note {document_id[:8]}... orphaned from collection — "
569 "skipping orphan delete; use DELETE /api/notes/<id> to remove."
570 )
571 elif remaining_count == 0:
572 # Document is orphaned - delete it completely
573 # Note: We're already in a session, so we need to do this
574 # directly rather than calling delete_document()
575 logger.info(
576 f"Document {document_id[:8]}... is orphaned, deleting"
577 )
579 # Full delete: purge vectors/chunks across EVERY collection
580 # that still holds chunk rows for this document, not just
581 # the one being removed (a partial reindex can strand
582 # chunks elsewhere). Rows are deleted in the post-commit
583 # phase so purge_document_vectors can resolve their ids.
584 chunk_collection_rows = (
585 session.query(DocumentChunk.collection_name)
586 .filter_by(
587 source_type="document", source_id=document_id
588 )
589 .distinct()
590 .all()
591 )
592 full_delete = True
593 rag_collection_ids = list(
594 {collection_id}
595 | {
596 row[0].removeprefix("collection_")
597 for row in chunk_collection_rows
598 if row[0]
599 }
600 )
602 # Update DownloadTracker
603 CascadeHelper.update_download_tracker(session, document)
605 # Delete filesystem file if applicable
606 if (
607 document.storage_mode == "filesystem"
608 and document.file_path
609 ):
610 from ...utils import get_absolute_path_from_settings
612 try:
613 file_path = get_absolute_path_from_settings(
614 document.file_path
615 )
616 if file_path:
617 CascadeHelper.delete_filesystem_file(
618 str(file_path)
619 )
620 except Exception:
621 logger.exception("Failed to delete filesystem file")
623 # Delete document and all related records (leaves the
624 # DocumentChunk rows for the post-commit cleanup phase).
625 CascadeHelper.delete_document_completely(
626 session, document_id
627 )
628 result["document_deleted"] = True
630 session.commit()
631 result["unlinked"] = True
633 logger.info(
634 f"Removed document {document_id[:8]}... from collection "
635 f"{collection_id[:8]}... "
636 f"(deleted={result['document_deleted']})"
637 )
639 except Exception:
640 logger.exception(
641 f"Failed to remove document {document_id} "
642 f"from collection {collection_id}"
643 )
644 session.rollback()
645 return {
646 "unlinked": False,
647 "document_deleted": False,
648 "document_id": document_id,
649 "collection_id": collection_id,
650 "error": "Failed to remove document from collection",
651 }
653 # Post-commit FAISS + chunk-row cleanup, after the transaction has
654 # committed and released SQLite's write lock.
655 self._purge_document_rag(
656 document_id, rag_collection_ids, full_delete=full_delete
657 )
659 return result
661 def get_deletion_preview(self, document_id: str) -> Dict[str, Any]:
662 """
663 Get a preview of what will be deleted.
665 Useful for showing the user what will happen before confirming.
667 Args:
668 document_id: ID of the document
670 Returns:
671 Dict with preview information
672 """
673 with get_user_db_session(self.username) as session:
674 document = session.get(Document, document_id)
675 if not document:
676 return {"found": False, "document_id": document_id}
678 collections = CascadeHelper.get_document_collections(
679 session, document_id
680 )
682 # Count chunks
683 total_chunks = (
684 session.query(DocumentChunk)
685 .filter(
686 DocumentChunk.source_id == document_id,
687 DocumentChunk.source_type == "document",
688 )
689 .count()
690 )
692 blob_size = CascadeHelper.get_document_blob_size(
693 session, document_id
694 )
696 return {
697 "found": True,
698 "document_id": document_id,
699 "title": document.title or document.filename or "Untitled",
700 "file_type": document.file_type,
701 "storage_mode": document.storage_mode,
702 "has_blob": blob_size > 0,
703 "blob_size": blob_size,
704 "has_text": bool(document.text_content),
705 "collections_count": len(collections),
706 "chunks_count": total_chunks,
707 }