Coverage for src/local_deep_research/vector_stores/legacy_rekey.py: 99%
184 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Login-time (phase-2) re-key of legacy RAG indices -- the per-user half of
2the vector-store cutover.
4``migrate_legacy_docstores`` (phase 1, ``legacy_cleanup.py``) runs once at
5server startup and turns each legacy ``<hash>.pkl`` docstore into a small,
6text-free ``<hash>.idmap.json`` sidecar (FAISS position -> uuid), deleting the
7plaintext ``.pkl``. It cannot go further: rebuilding the index keyed by
8``DocumentChunk.id`` needs the per-user encrypted DB (to resolve uuid ->
9``DocumentChunk.id``), which is only available once a user has logged in.
11This module is that second half. ``rekey_user_indexes`` is called once per
12user at login time (from ``database/encrypted_db.py``
13``_open_user_database_cold``, AFTER the user's DB connection is already
14cached -- see that function's docstring for why it must run there and not
15earlier). For each of the user's ``RAGIndex`` rows whose on-disk ``.faiss``
16still has a sidecar (i.e. phase 1 ran but phase 2 hasn't), it resolves
17``uuid -> DocumentChunk.id`` from the DB, re-keys the index in place via
18``legacy_cleanup.rekey_index_file`` (no re-embedding), and re-records file
19integrity. Any single index that fails is quarantined (moved aside) and its
20collection's indexed state is reset so the normal indexing path rebuilds it
21from scratch -- one bad index never blocks the rest of the user's indices,
22and this whole step never blocks login (the caller wraps it in try/except).
24Gated behind a one-time per-user settings marker
25(``research_library.rag_index_rekeyed_v2``) so this is a single
26``get_setting`` call (fast no-op) on every login after the first successful
27run, and for every user who has no legacy sidecars at all -- the common case
28once the fleet has passed through the cutover once.
29"""
31import json
32import time
33from pathlib import Path
34from typing import Dict, Iterable, List, Optional
36from loguru import logger
38from ..database.models.library import (
39 DocumentChunk,
40 DocumentCollection,
41 RAGIndex,
42 RagDocumentStatus,
43)
44from ..database.session_context import get_user_db_session
45from ..security.file_integrity import FAISSIndexVerifier, FileIntegrityManager
46from ..security.log_sanitizer import redact_secrets
47from ..utilities.db_utils import get_settings_manager
48from .legacy_cleanup import rekey_index_file
50# Settings key gating the one-time per-user re-key. Only set after every one
51# of the user's indices has been resolved (rekeyed, skipped, or
52# quarantined) -- see ``rekey_user_indexes`` for what happens if the process
53# is killed mid-run (next login just retries the remaining sidecars).
54REKEY_MARKER_SETTING = "research_library.rag_index_rekeyed_v2"
56# SQLITE_MAX_VARIABLE_NUMBER is 999 on older SQLite builds; chunk the
57# embedding_id IN() lookup well below it. Mirrors ``_IN_CLAUSE_CHUNK`` in
58# research_library/notes/services/note_service.py.
59_IN_CLAUSE_CHUNK = 500
61_SIDECAR_SUFFIX = ".idmap.json"
64def _sidecar_for(faiss_path: Path) -> Path:
65 # <hash>.faiss -> <hash>.idmap.json (same stem as legacy_cleanup uses
66 # for <hash>.pkl -> <hash>.idmap.json, since .pkl and .faiss share a stem).
67 return faiss_path.with_name(faiss_path.stem + _SIDECAR_SUFFIX)
70def _is_idmap2(faiss_path: Path) -> Optional[bool]:
71 """``True`` iff the ``.faiss`` loads as the new ``IndexIDMap2`` format;
72 ``False`` if it loads as a DIFFERENT (old) format; ``None`` if it could NOT
73 be read at all.
75 Used to tell an already-migrated index (IndexIDMap2, no sidecar) apart from
76 a phase-1 reindex-fallback leftover (an OLD-format base index whose ``.pkl``
77 was deleted without writing a sidecar). The caller quarantines ONLY on a
78 definite ``False``. A ``None`` (transient IO/permission error) must NOT
79 destructively quarantine + full-re-embed a possibly-healthy index — the
80 caller withholds the marker and retries next login instead. A genuinely
81 corrupt file that keeps failing is still caught by the normal index/search
82 load-failure path, which quarantines there.
83 """
84 from faiss import IndexIDMap2, read_index
86 try:
87 index = read_index(str(faiss_path))
88 except Exception:
89 return None
90 return isinstance(index, IndexIDMap2)
93def _load_uuid_to_id(
94 username: str, db_password: str, uuids: Iterable[str]
95) -> Dict[str, int]:
96 """Resolve ``DocumentChunk.embedding_id`` (uuid) -> ``DocumentChunk.id``
97 (int) for the given uuids, chunking the ``IN()`` list. uuids with no
98 matching row (deleted chunks) are simply absent from the result --
99 ``rekey_index_file`` drops those positions as orphans."""
100 uuid_list = list(uuids)
101 uuid_to_id: Dict[str, int] = {}
102 with get_user_db_session(username, db_password) as session:
103 for i in range(0, len(uuid_list), _IN_CLAUSE_CHUNK):
104 batch = uuid_list[i : i + _IN_CLAUSE_CHUNK]
105 rows = (
106 session.query(DocumentChunk.embedding_id, DocumentChunk.id)
107 .filter(DocumentChunk.embedding_id.in_(batch))
108 .all()
109 )
110 uuid_to_id.update(dict(rows))
111 return uuid_to_id
114def _resolve_index_params(rag_index: RAGIndex, settings) -> Dict:
115 """RAGIndex row values win; fall back to the current ``local_search_*``
116 defaults for legacy rows that predate these columns -- mirrors the
117 default resolution in ``rag_service_factory.get_rag_service``."""
118 return {
119 "dimension": rag_index.embedding_dimension,
120 "index_type": rag_index.index_type
121 or (settings.get_setting("local_search_index_type") or "flat"),
122 "metric": rag_index.distance_metric
123 or (settings.get_setting("local_search_distance_metric") or "cosine"),
124 "normalize": (
125 rag_index.normalize_vectors
126 if rag_index.normalize_vectors is not None
127 else settings.get_bool_setting("local_search_normalize_vectors")
128 ),
129 }
132def _persist_rekeyed_config(
133 username: str, db_password: str, rag_index_id: int, params: Dict
134) -> None:
135 """Pin the config the physical file was ACTUALLY built with onto the
136 RAGIndex row.
138 Legacy rows predate the index_type/distance_metric/normalize_vectors
139 columns, so ``_resolve_index_params`` fills them from the live
140 ``local_search_*`` settings and ``rekey_index_file`` bakes that choice into
141 the .faiss file. If the row is left NULL, every later access re-resolves the
142 SAME settings-fallback chain — so changing a global default afterward makes
143 the runtime reinterpret the file under a different index_type/metric/
144 normalize than it was built with. Write the resolved values back (only where
145 the row is still NULL — never override an explicit stored value)."""
146 with get_user_db_session(username, db_password) as session:
147 row = session.query(RAGIndex).filter_by(id=rag_index_id).first()
148 if row is None:
149 return
150 changed = False
151 if row.index_type is None:
152 row.index_type = params["index_type"]
153 changed = True
154 if row.distance_metric is None:
155 row.distance_metric = params["metric"]
156 changed = True
157 if row.normalize_vectors is None:
158 row.normalize_vectors = params["normalize"]
159 changed = True
160 if changed:
161 session.commit()
164def _purge_redundant_pkl(faiss_path: Path, db_password: str) -> bool:
165 """Delete a legacy plaintext ``.pkl`` that has become REDUNDANT.
167 Once phase-1 has extracted the sidecar (or the index is already an
168 IndexIDMap2), the ``.pkl``'s chunk text lives only in the encrypted DB and
169 the id-map lives in the sidecar, so the ``.pkl`` is pure leftover plaintext.
170 Phase-1 deletes it, but that one-time unlink can fail (a directory the app
171 can't write, a read-only mount, an AV/EDR lock) AFTER the sidecar was
172 written — leaving plaintext on disk that no later step revisits once the
173 per-user completion marker is set. Callers invoke this at every phase-2
174 finalization point (re-key success, already-migrated skip) as a backstop.
176 Returns True if the ``.pkl`` is absent or was deleted, False if deletion
177 failed (the caller then withholds the completion marker so phase-2 retries
178 instead of gating the user out with plaintext still on disk). Deliberately
179 NOT called where a ``.pkl`` is still NEEDED (no sidecar + old-format index,
180 i.e. phase-1 hasn't converted it yet) — that case is handled separately.
181 """
182 pkl = faiss_path.with_suffix(".pkl")
183 if not pkl.exists():
184 return True
185 try:
186 pkl.unlink()
187 logger.info(f"Removed surviving legacy plaintext .pkl {pkl}")
188 return True
189 except OSError as exc:
190 safe_msg = redact_secrets(str(exc), db_password)
191 logger.warning(
192 f"Could not remove surviving legacy plaintext .pkl {pkl}; "
193 "withholding re-key completion so it is retried next login: "
194 f"{safe_msg}"
195 )
196 return False
199def _quarantine_and_reset(
200 username: str,
201 db_password: str,
202 rag_index: RAGIndex,
203 faiss_path: Path,
204 sidecar_path: Path,
205 lock,
206) -> None:
207 """Move the broken index pair aside and reset the collection's indexed
208 state so the normal indexing path rebuilds it from scratch.
210 Mirrors ``LibraryRAGService._quarantine_corrupt_index`` +
211 ``_reset_index_state_for_rebuild`` in spirit (rename-aside, never
212 delete; clear ``RagDocumentStatus`` + ``DocumentCollection.indexed`` so
213 the next index run does real work instead of skipping everything).
214 Kept as a free function here (rather than reusing those bound methods
215 directly) so this module doesn't have to construct a full
216 ``LibraryRAGService`` -- which would pull in an embedding provider --
217 just to quarantine a broken re-key.
219 Raises on rename failure so the caller can log it distinctly and, in
220 turn, withhold the per-user completion marker (see
221 ``rekey_user_indexes``) so this index is retried on the next login
222 instead of being silently abandoned behind a "done" marker.
223 """
224 # Reset DB state FIRST, then move the files aside. These two steps are not
225 # atomic, so ordering decides what a crash in between leaves behind:
226 # - DB-first (here): the collection is durably flagged "needs reindex"
227 # before its file is touched. A crash after the commit leaves a stray
228 # (still-original or renamed) file that the next index run simply
229 # rebuilds over — no data lost, no wrong state.
230 # - file-first (the reverse) would strand the collection as "indexed" with
231 # its backing index renamed away and no recoverable data until an admin
232 # intervenes — the exact permanent-loss failure this ordering avoids.
233 collection_id = rag_index.collection_name.removeprefix("collection_")
234 with get_user_db_session(username, db_password) as session:
235 idx = session.query(RAGIndex).filter_by(id=rag_index.id).first()
236 if idx: 236 ↛ 239line 236 didn't jump to line 239 because the condition on line 236 was always true
237 idx.chunk_count = 0
238 idx.total_documents = 0
239 session.query(RagDocumentStatus).filter_by(
240 rag_index_id=rag_index.id
241 ).delete()
242 # Only the CURRENT index backs the collection's live searches, so only
243 # its failure should force the whole collection to re-embed. Quarantining
244 # a STALE (is_current=False) row must NOT flip the collection's live
245 # DocumentCollection.indexed state -- the current index is still healthy
246 # and serving searches; clearing this row's RAGIndex + RagDocumentStatus
247 # above is enough. (rekey_user_indexes now re-keys stale rows too, so
248 # this guard is what keeps a stale-row failure from needlessly
249 # re-embedding the whole collection.)
250 if idx is not None and idx.is_current:
251 session.query(DocumentCollection).filter_by(
252 collection_id=collection_id, indexed=True
253 ).update({"indexed": False, "chunk_count": 0})
254 session.commit()
256 # Hold the per-(user, index_path) write lock for the renames so they don't
257 # race a concurrent search/index reading the same .faiss (which would see
258 # the file vanish mid-operation).
259 ns = time.time_ns()
260 with lock:
261 if faiss_path.exists():
262 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Login rekey uses service-generated RAGIndex.index_path values from the authenticated user's DB; quarantine only appends a generated timestamp in that directory
263 faiss_path.rename(Path(f"{faiss_path}.corrupt-{ns}"))
264 if sidecar_path.exists():
265 # nosemgrep: semgrep.rules.path-traversal-risk, reason: The caller derives this fixed-suffix sidecar from the same service-generated index path; quarantine only appends a generated timestamp in that directory
266 sidecar_path.rename(Path(f"{sidecar_path}.corrupt-{ns}"))
267 # Also DELETE (not rename) any lingering legacy plaintext .pkl companion.
268 # Phase-1 normally removes it at startup, but if it couldn't then
269 # (unreadable/symlinked at that moment) or it reappeared, quarantining
270 # the .faiss must not leave the plaintext docstore on disk. Renaming
271 # aside like the .faiss would preserve the plaintext, so unlink it. Let
272 # an unlink failure PROPAGATE (like the renames above): swallowing it
273 # counted the quarantine as clean success and set the completion marker
274 # while plaintext survived on disk. The caller instead logs it and
275 # withholds the marker so phase-2 retries.
276 pkl = faiss_path.with_suffix(".pkl")
277 if pkl.exists():
278 pkl.unlink()
279 logger.warning(
280 f"Quarantined un-rekeyable RAG index {faiss_path} for collection "
281 f"{collection_id}; reset its indexed state so it rebuilds on next use."
282 )
285def rekey_user_indexes(username: str, db_password: str) -> Dict[str, int]:
286 """Re-key this user's legacy RAG indices (phase 2 of the cutover), once.
288 MUST be called only after ``username``'s DB connection is already in
289 ``db_manager.connections``. This function goes through
290 ``get_user_db_session`` / ``FileIntegrityManager``, both of which
291 re-enter ``db_manager.open_user_database`` on a thread-local
292 session-cache miss. Calling this from inside ``_open_user_database_cold``
293 BEFORE the connection is cached would deadlock: ``open_user_database``
294 serializes the cold-open behind a per-user ``threading.Lock`` (not
295 reentrant), which this same thread is already holding at that point.
297 Returns a stats dict (``checked``/``rekeyed``/``quarantined``/
298 ``skipped_no_sidecar``/``skipped_no_faiss``). Does not raise -- every
299 per-index failure is caught and quarantined; the caller wraps this call
300 anyway as defense in depth.
301 """
302 stats = {
303 "checked": 0,
304 "rekeyed": 0,
305 "quarantined": 0,
306 "skipped_no_sidecar": 0,
307 "skipped_no_faiss": 0,
308 }
310 with get_user_db_session(username, db_password) as session:
311 settings = get_settings_manager(db_session=session, username=username)
312 if settings.get_bool_setting(REKEY_MARKER_SETTING, False):
313 return stats # already done for this user -- fast no-op
314 # Re-key EVERY RAGIndex row phase-1 converted, not just the current one.
315 # A collection can have stale historical rows (is_current=False) from an
316 # earlier embedding-model switch, and those are NOT unreachable: reverting
317 # the embedding-model setting re-resolves a search to the stale row by
318 # index_hash (library_rag_service._get_or_create_rag_index looks the row
319 # up by hash, NOT is_current), and a still-old-format .faiss then fails to
320 # load and hard-fails search. Phase-1 already writes text-free sidecars
321 # for ALL of them, so phase-2 must re-key ALL of them. To keep a stale
322 # row's re-key FAILURE from collaterally wiping the healthy current
323 # index's indexed state, _quarantine_and_reset resets the collection-wide
324 # DocumentCollection.indexed flag ONLY for the current row (see there).
325 indexes: List[RAGIndex] = session.query(RAGIndex).all()
327 if not indexes:
328 settings.set_setting(REKEY_MARKER_SETTING, True)
329 return stats
331 # Lazy import: avoids a module-load cycle (this is a leaf import used
332 # only inside this function -- see rekey_index_file's docstring for the
333 # same pattern in legacy_cleanup.py).
334 from ..research_library.services.library_rag_service import (
335 _get_faiss_write_lock,
336 )
338 integrity_manager = None # built lazily -- only if a sidecar is found
339 # Only set the completion marker if every index was actually resolved
340 # (rekeyed / skipped / quarantined). If quarantining itself fails (e.g.
341 # disk full mid-rename), that index is left untouched and the marker is
342 # withheld so the NEXT login retries it, instead of silently abandoning
343 # it forever behind a "done" marker.
344 all_resolved = True
346 for rag_index in indexes:
347 stats["checked"] += 1
348 # LibraryRAGService owns all index_path writes: _get_index_path uses
349 # an operator cache root and hash-based names. No request or import
350 # endpoint accepts this column. Keep that boundary when adding writers;
351 # being stored in the DB alone would not make a path trustworthy.
352 faiss_path = Path(rag_index.index_path)
353 sidecar_path = _sidecar_for(faiss_path)
354 # Per-(user, index_path) write lock, held for every file mutation
355 # (re-key persist, quarantine renames) so none race a concurrent
356 # search/index on the same .faiss.
357 lock = _get_faiss_write_lock(username, str(faiss_path))
359 if not sidecar_path.exists():
360 # No sidecar means EITHER already fully migrated (the .faiss is an
361 # IndexIDMap2) OR phase-1 hit its reindex-fallback: a corrupt/
362 # malformed legacy .pkl was deleted WITHOUT writing a sidecar,
363 # leaving the OLD-format .faiss stranded. The latter would otherwise
364 # be skipped forever while the DB still claims the collection is
365 # indexed, so its search silently breaks. Distinguish by format and
366 # quarantine + reset the un-migratable one so it rebuilds. (This
367 # runs once — the whole function is per-user marker-gated.)
368 fmt = _is_idmap2(faiss_path) if faiss_path.exists() else True
369 if fmt is None:
370 # Could NOT read the .faiss to determine its format (transient
371 # IO/permission). Do NOT quarantine a possibly-healthy index —
372 # withhold the marker and retry next login.
373 logger.warning(
374 f"Could not read {faiss_path} to check its format; will "
375 "retry next login (NOT quarantining a possibly-healthy "
376 "index)."
377 )
378 all_resolved = False
379 continue
380 if faiss_path.exists() and fmt is False:
381 # A no-sidecar old-format .faiss has TWO causes, and only one is
382 # the "phase-1 gave up" (reindex-fallback) case that warrants a
383 # destructive quarantine. If the legacy plaintext .pkl companion
384 # STILL EXISTS, phase-1 simply hasn't converted this file yet
385 # (it ran before this collection existed / crashed early / a
386 # sibling process's phase-1 hasn't reached it) — its sidecar is
387 # still to come. Quarantining now would force a needless full
388 # re-embed instead of the cheap rekey phase-1 will enable, and it
389 # would delete an intact docstore out from under phase-1. Withhold
390 # the completion marker and retry next login, by which point
391 # phase-1 (which runs every startup) will have written the
392 # sidecar and removed the .pkl.
393 if faiss_path.with_suffix(".pkl").exists():
394 logger.warning(
395 f"Old-format index {faiss_path} still has its legacy "
396 ".pkl and no sidecar — phase-1 has not converted it yet; "
397 "will retry next login (NOT quarantining)."
398 )
399 all_resolved = False
400 continue
401 logger.warning(
402 f"Un-migratable old-format index {faiss_path} has no "
403 "sidecar (phase-1 reindex-fallback); quarantining so its "
404 "collection rebuilds."
405 )
406 try:
407 _quarantine_and_reset(
408 username,
409 db_password,
410 rag_index,
411 faiss_path,
412 sidecar_path,
413 lock,
414 )
415 stats["quarantined"] += 1
416 except Exception as exc:
417 # db_password is a live parameter of this function; a
418 # rendered traceback (loguru diagnose=True) would dump it.
419 # Drop the traceback and redact str(exc). See encrypted_db.
420 safe_msg = redact_secrets(str(exc), db_password)
421 logger.warning(
422 f"Failed to quarantine {faiss_path}; will retry next "
423 f"login: {safe_msg}"
424 )
425 all_resolved = False
426 continue
427 # Already migrated (IndexIDMap2, no sidecar). Backstop: purge any
428 # legacy plaintext .pkl that survived phase-1 here too — this is the
429 # retry landing spot after a re-key whose Phase-B .pkl unlink failed
430 # (sidecar already gone), so without it the marker would be set with
431 # plaintext still on disk.
432 if not _purge_redundant_pkl(faiss_path, db_password):
433 all_resolved = False
434 stats["skipped_no_sidecar"] += 1
435 continue
436 if not faiss_path.exists():
437 # Orphaned sidecar (index file itself is already gone) -- there
438 # is nothing to re-key. Purge any legacy plaintext .pkl that
439 # survived next to the now-gone .faiss (a phase-1 .pkl-unlink
440 # failure, or a partial _quarantine_and_reset that renamed the
441 # .faiss aside before reaching its own pkl.unlink, can leave one
442 # here) BEFORE dropping the sidecar. Every other finalization branch
443 # backstops the plaintext .pkl this way; without it this branch is
444 # the one path that can set the per-user completion marker while an
445 # encryption-defeating plaintext .pkl still sits on disk. Withhold
446 # the marker if EITHER cleanup fails so phase-2 retries next login.
447 stats["skipped_no_faiss"] += 1
448 if not _purge_redundant_pkl(faiss_path, db_password): 448 ↛ 449line 448 didn't jump to line 449 because the condition on line 448 was never true
449 all_resolved = False
450 try:
451 # nosemgrep: semgrep.rules.path-traversal-risk, reason: fixed-suffix sidecar of LibraryRAGService's generated index path loaded from the authenticated user's DB; see the index_path writer boundary above
452 sidecar_path.unlink(missing_ok=True)
453 except OSError:
454 logger.warning(
455 f"Could not remove orphaned sidecar {sidecar_path}; "
456 "will retry next login"
457 )
458 all_resolved = False
459 continue
461 # Resolve + PIN the index config onto the row BEFORE building the file,
462 # so the row records what the file is built with even across a
463 # crash/retry. Pinning AFTER a successful build would break the retry
464 # path: if settings change between a crashed attempt and the retry, the
465 # retry re-resolves DIFFERENT params, hits rekey_index_file's
466 # already-migrated short-circuit (which only checks dim + ids), and pins
467 # a config that disagrees with the physical file. Pinning FIRST makes the
468 # row the source of truth that _resolve_index_params reads on retry (row
469 # wins over settings), so the resolved params match the file at every
470 # crash point. A transient failure here must NOT quarantine a healthy
471 # legacy file — withhold the marker and retry next login.
472 try:
473 params = _resolve_index_params(rag_index, settings)
474 _persist_rekeyed_config(username, db_password, rag_index.id, params)
475 except Exception as exc:
476 safe_msg = redact_secrets(str(exc), db_password)
477 logger.warning(
478 f"Failed to pin index config for RAG index {rag_index.id} "
479 f"({faiss_path}); will retry next login: {safe_msg}"
480 )
481 all_resolved = False
482 continue
484 # Resolve the sidecar map + uuid->DocumentChunk.id (a DB read). A failure
485 # HERE is transient and unrelated to the index file's health — a
486 # 'database is locked' timeout under concurrent writes, or a momentary
487 # OSError reading the sidecar. It must NOT quarantine a perfectly healthy
488 # legacy index (which would force a needless full re-embed); withhold the
489 # marker and retry next login instead.
490 try:
491 # nosemgrep: semgrep.rules.path-traversal-risk, reason: fixed-suffix sidecar of LibraryRAGService's generated index path loaded from the authenticated user's DB; see the index_path writer boundary above
492 pos_to_uuid = json.loads(sidecar_path.read_text(encoding="utf-8"))
493 uuid_to_id = _load_uuid_to_id(
494 username, db_password, set(pos_to_uuid.values())
495 )
496 except Exception as exc:
497 safe_msg = redact_secrets(str(exc), db_password)
498 logger.warning(
499 f"Could not resolve sidecar/DB ids for RAG index "
500 f"{rag_index.id} ({faiss_path}); will retry next login "
501 f"(NOT quarantining — the index file is untouched): {safe_msg}"
502 )
503 all_resolved = False
504 continue
506 # Phase A -- re-key. A failure HERE is a genuinely un-rekeyable index
507 # (corrupt/foreign format, dimension mismatch, duplicate-id sidecar):
508 # quarantine + reset so the collection rebuilds from the DB.
509 try:
510 with lock:
511 rekey_index_file(faiss_path, sidecar_path, uuid_to_id, **params)
512 except Exception as exc:
513 safe_msg = redact_secrets(str(exc), db_password)
514 logger.warning(
515 f"Re-key failed for RAG index {rag_index.id} "
516 f"({faiss_path}); quarantining -- collection will reindex: "
517 f"{safe_msg}"
518 )
519 try:
520 _quarantine_and_reset(
521 username,
522 db_password,
523 rag_index,
524 faiss_path,
525 sidecar_path,
526 lock,
527 )
528 stats["quarantined"] += 1
529 except Exception as exc:
530 safe_msg = redact_secrets(str(exc), db_password)
531 logger.warning(
532 f"Failed to quarantine {faiss_path} after re-key "
533 f"failure -- leaving files as-is; will retry next login: "
534 f"{safe_msg}"
535 )
536 all_resolved = False
537 continue
539 # Phase B -- the re-key SUCCEEDED (byte-verified new IndexIDMap2).
540 # Record the new file's integrity, THEN delete the sidecar (the
541 # completion signal). A failure here must NOT quarantine a perfectly
542 # good index (the bug this split fixes): leave the sidecar so the next
543 # login retries -- rekey_index_file's IndexIDMap2 guard makes that a
544 # clean no-op re-key -- and withhold the completion marker.
545 try:
546 with lock:
547 if integrity_manager is None:
548 integrity_manager = FileIntegrityManager(
549 username, db_password
550 )
551 integrity_manager.register_verifier(FAISSIndexVerifier())
552 integrity_manager.record_file(
553 faiss_path,
554 related_entity_type="rag_index",
555 related_entity_id=rag_index.id,
556 )
557 # Config was already pinned before the build (see above), so
558 # Phase B only records integrity and drops the sidecar signal.
559 # nosemgrep: semgrep.rules.path-traversal-risk, reason: fixed-suffix sidecar of LibraryRAGService's generated index path loaded from the authenticated user's DB; see the index_path writer boundary above
560 sidecar_path.unlink(missing_ok=True)
561 # Backstop: if phase-1 wrote the sidecar but its .pkl unlink
562 # failed, the plaintext .pkl is still on disk. The re-key just
563 # succeeded from the sidecar and the text is in the DB, so purge
564 # the now-redundant .pkl; withhold the marker if it can't be
565 # removed so this index is retried rather than gated out with
566 # plaintext surviving.
567 if not _purge_redundant_pkl(faiss_path, db_password):
568 all_resolved = False
569 stats["rekeyed"] += 1
570 except Exception as exc:
571 safe_msg = redact_secrets(str(exc), db_password)
572 logger.warning(
573 f"Re-key of {faiss_path} succeeded but finalization "
574 "(integrity record / sidecar removal) failed; leaving sidecar "
575 f"to retry next login: {safe_msg}"
576 )
577 all_resolved = False
579 if all_resolved:
580 settings.set_setting(REKEY_MARKER_SETTING, True)
581 logger.info(
582 f"RAG index re-key for user {username}: "
583 f"checked={stats['checked']} rekeyed={stats['rekeyed']} "
584 f"quarantined={stats['quarantined']} "
585 f"skipped_no_sidecar={stats['skipped_no_sidecar']} "
586 f"skipped_no_faiss={stats['skipped_no_faiss']}"
587 + ("" if all_resolved else " (incomplete -- will retry next login)")
588 )
589 return stats