Coverage for src/local_deep_research/research_library/deletion/services/collection_deletion.py: 98%
107 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-20 01:24 +0000
1"""
2Collection deletion service.
4Handles:
5- Full collection deletion with proper cleanup
6- Documents are preserved but unlinked
7- RAG index and chunks are deleted
8"""
10from typing import Dict, List, Any
12from loguru import logger
14from ....database.models.library import (
15 Collection,
16 Document,
17 DocumentCollection,
18 DocumentChunk,
19 CollectionFolder,
20 RAGIndex,
21 RagDocumentStatus,
22 SourceType,
23)
24from ....database.session_context import get_user_db_session
25from ..utils.cascade_helper import CascadeHelper
27# System collection types that must never be deleted via the public API.
28# These collections hold first-class user data whose lifecycle is owned by
29# other subsystems (Library, Research History, Notes); allowing them to be
30# deleted via the generic collection-delete path triggers cascade orphan
31# deletion of every document inside, which for Notes means total data loss
32# of notes, versions, links, and synthesis sources.
33# NOTE: 'zotero' is deliberately NOT protected — unlike the singleton system
34# collections above, users create/remove Zotero-synced collections and the
35# generic delete path is their only UI for it. Whether deleting a Zotero
36# collection should also purge its synced documents is a separate question
37# for the Zotero sync owner to decide, not something to block wholesale here.
38PROTECTED_COLLECTION_TYPES = frozenset(
39 {"default_library", "research_history", "notes"}
40)
43class CollectionDeletionService:
44 """Service for collection deletion operations."""
46 def __init__(self, username: str):
47 """
48 Initialize collection deletion service.
50 Args:
51 username: Username for database session
52 """
53 self.username = username
55 def delete_collection(
56 self, collection_id: str, delete_orphaned_documents: bool = True
57 ) -> Dict[str, Any]:
58 """
59 Delete a collection and clean up all related data.
61 By default, orphaned documents (not in any other collection) are deleted.
62 Set delete_orphaned_documents=False to preserve all documents.
64 The following are deleted:
65 - DocumentChunks for this collection
66 - FAISS index files
67 - RAGIndex records
68 - CollectionFolder records (CASCADE)
69 - DocumentCollection links (CASCADE)
70 - RagDocumentStatus records (CASCADE)
71 - Orphaned documents (if delete_orphaned_documents=True)
73 Args:
74 collection_id: ID of the collection to delete
75 delete_orphaned_documents: If True, delete documents not in any
76 other collection after unlinking
78 Returns:
79 Dict with deletion details:
80 {
81 "deleted": True/False,
82 "collection_id": str,
83 "collection_name": str,
84 "chunks_deleted": int,
85 "documents_unlinked": int,
86 "indices_deleted": int,
87 "folders_deleted": int,
88 "orphaned_documents_deleted": int,
89 "error": str (if failed)
90 }
91 """
92 with get_user_db_session(self.username) as session:
93 try:
94 # Get collection
95 collection = session.get(Collection, collection_id)
96 if not collection:
97 return {
98 "deleted": False,
99 "collection_id": collection_id,
100 "error": "Collection not found",
101 }
103 if collection.collection_type in PROTECTED_COLLECTION_TYPES:
104 logger.warning(
105 "Refused to delete protected system collection "
106 f"{collection_id[:8]}... (type={collection.collection_type})"
107 )
108 return {
109 "deleted": False,
110 "collection_id": collection_id,
111 "collection_name": collection.name,
112 "collection_type": collection.collection_type,
113 "error": (
114 "Cannot delete system collection "
115 f"'{collection.name}' (type={collection.collection_type}). "
116 "This collection holds first-class user data."
117 ),
118 }
120 collection_name = f"collection_{collection_id}"
121 result = {
122 "deleted": False,
123 "collection_id": collection_id,
124 "collection_name": collection.name,
125 "chunks_deleted": 0,
126 "documents_unlinked": 0,
127 "indices_deleted": 0,
128 "folders_deleted": 0,
129 "orphaned_documents_deleted": 0,
130 }
132 # 1. Get document IDs BEFORE deleting links (for orphan check)
133 doc_ids_in_collection = [
134 dc.document_id
135 for dc in session.query(DocumentCollection)
136 .filter_by(collection_id=collection_id)
137 .all()
138 ]
139 result["documents_unlinked"] = len(doc_ids_in_collection)
141 # 2. Delete DocumentChunks for this collection
142 result["chunks_deleted"] = (
143 CascadeHelper.delete_collection_chunks(
144 session, collection_name
145 )
146 )
148 # 3. Delete RAGIndex records (DB only). The on-disk FAISS files
149 # are unlinked AFTER the commit below — steps 4-8 still do DB
150 # work that could raise and roll back, and a rollback restores
151 # the RAGIndex rows but not files already deleted from disk.
152 rag_result = CascadeHelper.delete_rag_indices_for_collection(
153 session, collection_name, unlink_files=False
154 )
155 result["indices_deleted"] = rag_result["deleted_indices"]
156 faiss_paths_to_unlink = rag_result["index_paths"]
157 # Orphaned documents hard-deleted below may still have chunk
158 # rows/vectors in OTHER collections (a partial reindex can strand
159 # them). Collect them here and purge those after the commit —
160 # delete_document_completely does not touch DocumentChunk.
161 orphaned_doc_ids: List[str] = []
163 # 4. Count folders before deletion
164 result["folders_deleted"] = (
165 session.query(CollectionFolder)
166 .filter_by(collection_id=collection_id)
167 .count()
168 )
170 # 5. Delete DocumentCollection links explicitly before collection
171 session.query(DocumentCollection).filter_by(
172 collection_id=collection_id
173 ).delete(synchronize_session=False)
175 # 6. Delete linked folders explicitly
176 session.query(CollectionFolder).filter_by(
177 collection_id=collection_id
178 ).delete(synchronize_session=False)
180 # 7. Delete the collection itself
181 session.delete(collection)
183 # 8. Delete orphaned documents if requested
184 if delete_orphaned_documents:
185 # Notes must never be hard-deleted via the orphan
186 # cascade: they live behind their own deletion API
187 # (DELETE /api/notes/<id>) and remain discoverable
188 # via list_notes(), which filters by source_type_id
189 # rather than collection membership. This mirrors the
190 # note-skip in document_deletion.py's orphan path. The
191 # check is kept local (a single SourceType lookup) to
192 # avoid importing the notes-services package into the
193 # deletion package, matching document_deletion's own
194 # local _is_note_document.
195 note_source = (
196 session.query(SourceType).filter_by(name="note").first()
197 )
198 for doc_id in doc_ids_in_collection:
199 # Check if document is in any other collection
200 remaining = (
201 session.query(DocumentCollection)
202 .filter_by(document_id=doc_id)
203 .count()
204 )
205 if remaining == 0:
206 if note_source is not None:
207 document = session.get(Document, doc_id)
208 if (
209 document is not None
210 and document.source_type_id
211 == note_source.id
212 ):
213 logger.info(
214 f"Note {doc_id[:8]}... orphaned by "
215 "collection deletion — skipping orphan "
216 "delete; use DELETE /api/notes/<id> "
217 "to remove."
218 )
219 continue
220 # Document is orphaned - delete it
221 CascadeHelper.delete_document_completely(
222 session, doc_id
223 )
224 orphaned_doc_ids.append(doc_id)
225 result["orphaned_documents_deleted"] += 1
226 logger.info(
227 f"Deleted orphaned document {doc_id[:8]}..."
228 )
230 session.commit()
232 result["deleted"] = True
233 logger.info(
234 f"Deleted collection {collection_id[:8]}... "
235 f"({result['collection_name']}): {result['chunks_deleted']} chunks, "
236 f"{result['documents_unlinked']} documents unlinked, "
237 f"{result['orphaned_documents_deleted']} orphaned deleted"
238 )
240 except Exception:
241 logger.exception(f"Failed to delete collection {collection_id}")
242 session.rollback()
243 return {
244 "deleted": False,
245 "collection_id": collection_id,
246 "error": "Failed to delete collection",
247 }
249 # Post-commit best-effort cleanup, OUTSIDE the deletion transaction and
250 # its try/except: the DB delete is already durable, so a failure here
251 # must NOT be reported as a failed collection delete — it isn't one. It
252 # would only leave orphaned files/vectors, logged for ops visibility.
253 for faiss_path in faiss_paths_to_unlink:
254 CascadeHelper.delete_faiss_index_files(faiss_path)
256 # Purge each hard-deleted orphan's vectors + chunk rows in its OTHER
257 # collections (this collection's chunks were removed in step 2). Reuses
258 # DocumentDeletionService's per-collection vector purge + chunk backstop.
259 if orphaned_doc_ids:
260 from .document_deletion import DocumentDeletionService
261 from ....database.models.library import DocumentChunk
263 deleter = DocumentDeletionService(self.username)
264 for doc_id in orphaned_doc_ids:
265 try:
266 with get_user_db_session(self.username) as purge_session:
267 coll_rows = (
268 purge_session.query(DocumentChunk.collection_name)
269 .filter_by(source_type="document", source_id=doc_id)
270 .distinct()
271 .all()
272 )
273 collection_ids = [
274 row[0].removeprefix("collection_")
275 for row in coll_rows
276 if row[0]
277 ]
278 deleter._purge_document_rag(
279 doc_id, collection_ids, full_delete=True
280 )
281 except Exception:
282 logger.opt(exception=True).error(
283 "Failed to purge orphaned document {} after collection "
284 "delete (delete already committed)",
285 doc_id,
286 )
288 return result
290 def delete_collection_index_only(
291 self, collection_id: str
292 ) -> Dict[str, Any]:
293 """
294 Delete only the RAG index for a collection, keeping the collection itself.
296 This is useful for rebuilding an index from scratch.
298 Args:
299 collection_id: ID of the collection
301 Returns:
302 Dict with deletion details
303 """
304 with get_user_db_session(self.username) as session:
305 try:
306 # Verify collection exists
307 collection = session.get(Collection, collection_id)
308 if not collection:
309 return {
310 "deleted": False,
311 "collection_id": collection_id,
312 "error": "Collection not found",
313 }
315 collection_name = f"collection_{collection_id}"
316 result = {
317 "deleted": False,
318 "collection_id": collection_id,
319 "chunks_deleted": 0,
320 "indices_deleted": 0,
321 "documents_reset": 0,
322 }
324 # 1. Delete DocumentChunks
325 result["chunks_deleted"] = (
326 CascadeHelper.delete_collection_chunks(
327 session, collection_name
328 )
329 )
331 # 2. Delete RAGIndex records (DB only). Files are unlinked after
332 # the commit — steps 3-5 do more DB work that could roll back,
333 # and a rollback can't restore files already removed from disk.
334 rag_result = CascadeHelper.delete_rag_indices_for_collection(
335 session, collection_name, unlink_files=False
336 )
337 result["indices_deleted"] = rag_result["deleted_indices"]
338 faiss_paths_to_unlink = rag_result["index_paths"]
340 # 3. Reset DocumentCollection indexed status
341 result["documents_reset"] = (
342 session.query(DocumentCollection)
343 .filter_by(collection_id=collection_id)
344 .update({"indexed": False, "chunk_count": 0})
345 )
347 # 4. Delete RagDocumentStatus for this collection
348 session.query(RagDocumentStatus).filter_by(
349 collection_id=collection_id
350 ).delete(synchronize_session=False)
352 # 5. Reset collection embedding info
353 collection.embedding_model = None
354 collection.embedding_model_type = None
355 collection.embedding_dimension = None
356 collection.chunk_size = None
357 collection.chunk_overlap = None
359 session.commit()
361 # DB delete durable — now unlink the FAISS files (best-effort).
362 for faiss_path in faiss_paths_to_unlink:
363 CascadeHelper.delete_faiss_index_files(faiss_path)
365 result["deleted"] = True
367 logger.info(
368 f"Deleted index for collection {collection_id[:8]}...: "
369 f"{result['chunks_deleted']} chunks, "
370 f"{result['documents_reset']} documents reset"
371 )
373 return result
375 except Exception:
376 logger.exception(
377 f"Failed to delete index for collection {collection_id}"
378 )
379 session.rollback()
380 return {
381 "deleted": False,
382 "collection_id": collection_id,
383 "error": "Failed to delete collection index",
384 }
386 def get_deletion_preview(self, collection_id: str) -> Dict[str, Any]:
387 """
388 Get a preview of what will be deleted.
390 Useful for showing the user what will happen before confirming.
392 Args:
393 collection_id: ID of the collection
395 Returns:
396 Dict with preview information
397 """
398 with get_user_db_session(self.username) as session:
399 collection = session.get(Collection, collection_id)
400 if not collection:
401 return {"found": False, "collection_id": collection_id}
403 collection_name = f"collection_{collection_id}"
405 # Count documents
406 documents_count = (
407 session.query(DocumentCollection)
408 .filter_by(collection_id=collection_id)
409 .count()
410 )
412 # Count chunks
413 chunks_count = (
414 session.query(DocumentChunk)
415 .filter_by(collection_name=collection_name)
416 .count()
417 )
419 # Count folders
420 folders_count = (
421 session.query(CollectionFolder)
422 .filter_by(collection_id=collection_id)
423 .count()
424 )
426 # Check for RAG index
427 has_index = (
428 session.query(RAGIndex)
429 .filter_by(collection_name=collection_name)
430 .first()
431 is not None
432 )
434 return {
435 "found": True,
436 "collection_id": collection_id,
437 "name": collection.name,
438 "description": collection.description,
439 "is_default": collection.is_default,
440 "documents_count": documents_count,
441 "chunks_count": chunks_count,
442 "folders_count": folders_count,
443 "has_rag_index": has_index,
444 "embedding_model": collection.embedding_model,
445 }