Coverage for src/local_deep_research/research_library/deletion/services/collection_deletion.py: 99%
113 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""
2Collection deletion service.
4Handles:
5- Full collection deletion with proper cleanup
6- Documents are preserved but unlinked
7- RAG index and chunks are deleted
8"""
10from typing import Dict, List, Any
12from loguru import logger
14from ....database.models.library import (
15 Collection,
16 Document,
17 DocumentCollection,
18 DocumentChunk,
19 CollectionFolder,
20 RAGIndex,
21 RagDocumentStatus,
22 SourceType,
23)
24from ....database.session_context import get_user_db_session
25from ..utils.cascade_helper import CascadeHelper
27# System collection types that must never be deleted via the public API.
28# These collections hold first-class user data whose lifecycle is owned by
29# other subsystems (Library, Research History, Notes); allowing them to be
30# deleted via the generic collection-delete path triggers cascade orphan
31# deletion of every document inside, which for Notes means total data loss
32# of notes, versions, links, and synthesis sources.
33# NOTE: 'zotero' is deliberately NOT protected — unlike the singleton system
34# collections above, users create/remove Zotero-synced collections and the
35# generic delete path is their only UI for it. Whether deleting a Zotero
36# collection should also purge its synced documents is a separate question
37# for the Zotero sync owner to decide, not something to block wholesale here.
38PROTECTED_COLLECTION_TYPES = frozenset(
39 {"default_library", "research_history", "notes"}
40)
43class CollectionDeletionService:
44 """Service for collection deletion operations."""
46 def __init__(self, username: str):
47 """
48 Initialize collection deletion service.
50 Args:
51 username: Username for database session
52 """
53 self.username = username
55 def delete_collection(
56 self, collection_id: str, delete_orphaned_documents: bool = True
57 ) -> Dict[str, Any]:
58 """
59 Delete a collection and clean up all related data.
61 By default, orphaned documents (not in any other collection) are deleted.
62 Set delete_orphaned_documents=False to preserve all documents.
64 The following are deleted:
65 - DocumentChunks for this collection
66 - FAISS index files
67 - RAGIndex records
68 - CollectionFolder records (CASCADE)
69 - DocumentCollection links (CASCADE)
70 - RagDocumentStatus records (CASCADE)
71 - Orphaned documents (if delete_orphaned_documents=True)
73 Args:
74 collection_id: ID of the collection to delete
75 delete_orphaned_documents: If True, delete documents not in any
76 other collection after unlinking
78 Returns:
79 Dict with deletion details:
80 {
81 "deleted": True/False,
82 "collection_id": str,
83 "collection_name": str,
84 "chunks_deleted": int,
85 "documents_unlinked": int,
86 "indices_deleted": int,
87 "folders_deleted": int,
88 "orphaned_documents_deleted": int,
89 "error": str (if failed)
90 }
91 """
92 with get_user_db_session(self.username) as session:
93 try:
94 # Get collection
95 collection = session.get(Collection, collection_id)
96 if not collection:
97 return {
98 "deleted": False,
99 "collection_id": collection_id,
100 "error": "Collection not found",
101 }
103 if collection.collection_type in PROTECTED_COLLECTION_TYPES:
104 logger.warning(
105 "Refused to delete protected system collection "
106 f"{collection_id[:8]}... (type={collection.collection_type})"
107 )
108 return {
109 "deleted": False,
110 "collection_id": collection_id,
111 "collection_name": collection.name,
112 "collection_type": collection.collection_type,
113 "error": (
114 "Cannot delete system collection "
115 f"'{collection.name}' (type={collection.collection_type}). "
116 "This collection holds first-class user data."
117 ),
118 }
120 collection_name = f"collection_{collection_id}"
121 result = {
122 "deleted": False,
123 "collection_id": collection_id,
124 "collection_name": collection.name,
125 "chunks_deleted": 0,
126 "documents_unlinked": 0,
127 "indices_deleted": 0,
128 "folders_deleted": 0,
129 "orphaned_documents_deleted": 0,
130 }
132 # 1. Get document IDs BEFORE deleting links (for orphan check)
133 doc_ids_in_collection = [
134 dc.document_id
135 for dc in session.query(DocumentCollection)
136 .filter_by(collection_id=collection_id)
137 .all()
138 ]
139 result["documents_unlinked"] = len(doc_ids_in_collection)
141 # 2. Delete DocumentChunks for this collection
142 result["chunks_deleted"] = (
143 CascadeHelper.delete_collection_chunks(
144 session, collection_name
145 )
146 )
148 # 3. Delete RAGIndex records (DB only). The on-disk FAISS files
149 # are unlinked AFTER the commit below — steps 4-8 still do DB
150 # work that could raise and roll back, and a rollback restores
151 # the RAGIndex rows but not files already deleted from disk.
152 rag_result = CascadeHelper.delete_rag_indices_for_collection(
153 session, collection_name, unlink_files=False
154 )
155 result["indices_deleted"] = rag_result["deleted_indices"]
156 faiss_paths_to_unlink = rag_result["index_paths"]
157 # Orphaned documents hard-deleted below may still have chunk
158 # rows/vectors in OTHER collections (a partial reindex can strand
159 # them). Collect them here and purge those after the commit —
160 # delete_document_completely does not touch DocumentChunk.
161 orphaned_doc_ids: List[str] = []
163 # 4. Count folders before deletion
164 result["folders_deleted"] = (
165 session.query(CollectionFolder)
166 .filter_by(collection_id=collection_id)
167 .count()
168 )
170 # 5. Delete DocumentCollection links explicitly before collection
171 session.query(DocumentCollection).filter_by(
172 collection_id=collection_id
173 ).delete(synchronize_session=False)
175 # 6. Delete linked folders explicitly
176 session.query(CollectionFolder).filter_by(
177 collection_id=collection_id
178 ).delete(synchronize_session=False)
180 # 7. Delete the collection itself
181 session.delete(collection)
183 # 8. Delete orphaned documents if requested
184 if delete_orphaned_documents:
185 # Notes must never be hard-deleted via the orphan
186 # cascade: they live behind their own deletion API
187 # (DELETE /api/notes/<id>) and remain discoverable
188 # via list_notes(), which filters by source_type_id
189 # rather than collection membership. This mirrors the
190 # note-skip in document_deletion.py's orphan path. The
191 # check is kept local (a single SourceType lookup) to
192 # avoid importing the notes-services package into the
193 # deletion package, matching document_deletion's own
194 # local _is_note_document.
195 note_source = (
196 session.query(SourceType).filter_by(name="note").first()
197 )
198 for doc_id in doc_ids_in_collection:
199 # Check if document is in any other collection
200 remaining = (
201 session.query(DocumentCollection)
202 .filter_by(document_id=doc_id)
203 .count()
204 )
205 if remaining == 0:
206 if note_source is not None:
207 document = session.get(Document, doc_id)
208 if (
209 document is not None
210 and document.source_type_id
211 == note_source.id
212 ):
213 logger.info(
214 f"Note {doc_id[:8]}... orphaned by "
215 "collection deletion — skipping orphan "
216 "delete; use DELETE /api/notes/<id> "
217 "to remove."
218 )
219 continue
220 # Document is orphaned - delete it
221 CascadeHelper.delete_document_completely(
222 session, doc_id
223 )
224 orphaned_doc_ids.append(doc_id)
225 result["orphaned_documents_deleted"] += 1
226 logger.info(
227 f"Deleted orphaned document {doc_id[:8]}..."
228 )
230 session.commit()
232 result["deleted"] = True
233 logger.info(
234 f"Deleted collection {collection_id[:8]}... "
235 f"({result['collection_name']}): {result['chunks_deleted']} chunks, "
236 f"{result['documents_unlinked']} documents unlinked, "
237 f"{result['orphaned_documents_deleted']} orphaned deleted"
238 )
240 except Exception:
241 logger.exception(f"Failed to delete collection {collection_id}")
242 session.rollback()
243 return {
244 "deleted": False,
245 "collection_id": collection_id,
246 "error": "Failed to delete collection",
247 }
249 # Post-commit best-effort cleanup, OUTSIDE the deletion transaction and
250 # its try/except: the DB delete is already durable, so a failure here
251 # must NOT be reported as a failed collection delete — it isn't one. It
252 # would only leave orphaned files/vectors, logged for ops visibility.
253 if faiss_paths_to_unlink:
254 from ....config.paths import get_cache_directory
256 # NOTE: allowed_root is the COARSE shared rag_indices/ root, not
257 # the tighter per-user rag_indices/<sha256(user)>/ subdir. It
258 # cannot be narrowed to the per-user subdir without risking
259 # refused deletes: pre-per-user-scoping (legacy) indexes live
260 # DIRECTLY in this shared root (see
261 # library_rag_service._migrate_legacy_index_files), so a per-user
262 # allowed_root would reject a legacy-layout RAGIndex.index_path
263 # and orphan its files. The containment check still blocks
264 # symlink / ancestor escapes outside rag_indices/ entirely.
265 rag_indices_root = get_cache_directory() / "rag_indices"
266 for faiss_path in faiss_paths_to_unlink:
267 CascadeHelper.delete_faiss_index_files(
268 faiss_path, allowed_root=rag_indices_root
269 )
271 # Purge each hard-deleted orphan's vectors + chunk rows in its OTHER
272 # collections (this collection's chunks were removed in step 2). Reuses
273 # DocumentDeletionService's per-collection vector purge + chunk backstop.
274 if orphaned_doc_ids:
275 from .document_deletion import DocumentDeletionService
276 from ....database.models.library import DocumentChunk
278 deleter = DocumentDeletionService(self.username)
279 for doc_id in orphaned_doc_ids:
280 try:
281 with get_user_db_session(self.username) as purge_session:
282 coll_rows = (
283 purge_session.query(DocumentChunk.collection_name)
284 .filter_by(source_type="document", source_id=doc_id)
285 .distinct()
286 .all()
287 )
288 collection_ids = [
289 row[0].removeprefix("collection_")
290 for row in coll_rows
291 if row[0]
292 ]
293 deleter._purge_document_rag(
294 doc_id, collection_ids, full_delete=True
295 )
296 except Exception:
297 logger.opt(exception=True).error(
298 "Failed to purge orphaned document {} after collection "
299 "delete (delete already committed)",
300 doc_id,
301 )
303 return result
305 def delete_collection_index_only(
306 self, collection_id: str
307 ) -> Dict[str, Any]:
308 """
309 Delete only the RAG index for a collection, keeping the collection itself.
311 This is useful for rebuilding an index from scratch.
313 Args:
314 collection_id: ID of the collection
316 Returns:
317 Dict with deletion details
318 """
319 with get_user_db_session(self.username) as session:
320 try:
321 # Verify collection exists
322 collection = session.get(Collection, collection_id)
323 if not collection:
324 return {
325 "deleted": False,
326 "collection_id": collection_id,
327 "error": "Collection not found",
328 }
330 collection_name = f"collection_{collection_id}"
331 result = {
332 "deleted": False,
333 "collection_id": collection_id,
334 "chunks_deleted": 0,
335 "indices_deleted": 0,
336 "documents_reset": 0,
337 }
339 # 1. Delete DocumentChunks
340 result["chunks_deleted"] = (
341 CascadeHelper.delete_collection_chunks(
342 session, collection_name
343 )
344 )
346 # 2. Delete RAGIndex records (DB only). Files are unlinked after
347 # the commit — steps 3-5 do more DB work that could roll back,
348 # and a rollback can't restore files already removed from disk.
349 rag_result = CascadeHelper.delete_rag_indices_for_collection(
350 session, collection_name, unlink_files=False
351 )
352 result["indices_deleted"] = rag_result["deleted_indices"]
353 faiss_paths_to_unlink = rag_result["index_paths"]
355 # 3. Reset DocumentCollection indexed status
356 result["documents_reset"] = (
357 session.query(DocumentCollection)
358 .filter_by(collection_id=collection_id)
359 .update({"indexed": False, "chunk_count": 0})
360 )
362 # 4. Delete RagDocumentStatus for this collection
363 session.query(RagDocumentStatus).filter_by(
364 collection_id=collection_id
365 ).delete(synchronize_session=False)
367 # 5. Reset collection embedding info
368 collection.embedding_model = None
369 collection.embedding_model_type = None
370 collection.embedding_dimension = None
371 collection.chunk_size = None
372 collection.chunk_overlap = None
374 session.commit()
376 # DB delete durable — now unlink the FAISS files (best-effort).
377 if faiss_paths_to_unlink:
378 from ....config.paths import get_cache_directory
380 # NOTE: allowed_root is the COARSE shared rag_indices/
381 # root, not the per-user rag_indices/<sha256(user)>/
382 # subdir -- see delete_collection above for why it can't
383 # be narrowed (legacy indexes live directly in the shared
384 # root, so a per-user scope would orphan them).
385 rag_indices_root = get_cache_directory() / "rag_indices"
386 for faiss_path in faiss_paths_to_unlink:
387 CascadeHelper.delete_faiss_index_files(
388 faiss_path, allowed_root=rag_indices_root
389 )
391 result["deleted"] = True
393 logger.info(
394 f"Deleted index for collection {collection_id[:8]}...: "
395 f"{result['chunks_deleted']} chunks, "
396 f"{result['documents_reset']} documents reset"
397 )
399 return result
401 except Exception:
402 logger.exception(
403 f"Failed to delete index for collection {collection_id}"
404 )
405 session.rollback()
406 return {
407 "deleted": False,
408 "collection_id": collection_id,
409 "error": "Failed to delete collection index",
410 }
412 def get_deletion_preview(self, collection_id: str) -> Dict[str, Any]:
413 """
414 Get a preview of what will be deleted.
416 Useful for showing the user what will happen before confirming.
418 Args:
419 collection_id: ID of the collection
421 Returns:
422 Dict with preview information
423 """
424 with get_user_db_session(self.username) as session:
425 collection = session.get(Collection, collection_id)
426 if not collection:
427 return {"found": False, "collection_id": collection_id}
429 collection_name = f"collection_{collection_id}"
431 # Count documents
432 documents_count = (
433 session.query(DocumentCollection)
434 .filter_by(collection_id=collection_id)
435 .count()
436 )
438 # Count chunks
439 chunks_count = (
440 session.query(DocumentChunk)
441 .filter_by(collection_name=collection_name)
442 .count()
443 )
445 # Count folders
446 folders_count = (
447 session.query(CollectionFolder)
448 .filter_by(collection_id=collection_id)
449 .count()
450 )
452 # Check for RAG index
453 has_index = (
454 session.query(RAGIndex)
455 .filter_by(collection_name=collection_name)
456 .first()
457 is not None
458 )
460 return {
461 "found": True,
462 "collection_id": collection_id,
463 "name": collection.name,
464 "description": collection.description,
465 "is_default": collection.is_default,
466 "documents_count": documents_count,
467 "chunks_count": chunks_count,
468 "folders_count": folders_count,
469 "has_rag_index": has_index,
470 "embedding_model": collection.embedding_model,
471 }