Coverage for src/local_deep_research/vector_stores/legacy_rekey.py: 73%
184 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-20 07:11 +0000
1"""Login-time (phase-2) re-key of legacy RAG indices -- the per-user half of
2the vector-store cutover.
4``migrate_legacy_docstores`` (phase 1, ``legacy_cleanup.py``) runs once at
5server startup and turns each legacy ``<hash>.pkl`` docstore into a small,
6text-free ``<hash>.idmap.json`` sidecar (FAISS position -> uuid), deleting the
7plaintext ``.pkl``. It cannot go further: rebuilding the index keyed by
8``DocumentChunk.id`` needs the per-user encrypted DB (to resolve uuid ->
9``DocumentChunk.id``), which is only available once a user has logged in.
11This module is that second half. ``rekey_user_indexes`` is called once per
12user at login time (from ``database/encrypted_db.py``
13``_open_user_database_cold``, AFTER the user's DB connection is already
14cached -- see that function's docstring for why it must run there and not
15earlier). For each of the user's ``RAGIndex`` rows whose on-disk ``.faiss``
16still has a sidecar (i.e. phase 1 ran but phase 2 hasn't), it resolves
17``uuid -> DocumentChunk.id`` from the DB, re-keys the index in place via
18``legacy_cleanup.rekey_index_file`` (no re-embedding), and re-records file
19integrity. Any single index that fails is quarantined (moved aside) and its
20collection's indexed state is reset so the normal indexing path rebuilds it
21from scratch -- one bad index never blocks the rest of the user's indices,
22and this whole step never blocks login (the caller wraps it in try/except).
24Gated behind a one-time per-user settings marker
25(``research_library.rag_index_rekeyed_v2``) so this is a single
26``get_setting`` call (fast no-op) on every login after the first successful
27run, and for every user who has no legacy sidecars at all -- the common case
28once the fleet has passed through the cutover once.
29"""
31import json
32import time
33from pathlib import Path
34from typing import Dict, Iterable, List, Optional, cast
36from loguru import logger
38from ..database.models.library import (
39 DocumentChunk,
40 DocumentCollection,
41 RAGIndex,
42 RagDocumentStatus,
43)
44from ..database.session_context import get_user_db_session
45from ..security.file_integrity import FAISSIndexVerifier, FileIntegrityManager
46from ..security.log_sanitizer import redact_secrets
47from ..utilities.db_utils import get_settings_manager
48from .legacy_cleanup import rekey_index_file
50# Settings key gating the one-time per-user re-key. Only set after every one
51# of the user's indices has been resolved (rekeyed, skipped, or
52# quarantined) -- see ``rekey_user_indexes`` for what happens if the process
53# is killed mid-run (next login just retries the remaining sidecars).
54REKEY_MARKER_SETTING = "research_library.rag_index_rekeyed_v2"
56# SQLITE_MAX_VARIABLE_NUMBER is 999 on older SQLite builds; chunk the
57# embedding_id IN() lookup well below it. Mirrors ``_IN_CLAUSE_CHUNK`` in
58# research_library/notes/services/note_service.py.
59_IN_CLAUSE_CHUNK = 500
61_SIDECAR_SUFFIX = ".idmap.json"
64def _sidecar_for(faiss_path: Path) -> Path:
65 # <hash>.faiss -> <hash>.idmap.json (same stem as legacy_cleanup uses
66 # for <hash>.pkl -> <hash>.idmap.json, since .pkl and .faiss share a stem).
67 return faiss_path.with_name(faiss_path.stem + _SIDECAR_SUFFIX)
70def _is_idmap2(faiss_path: Path) -> Optional[bool]:
71 """``True`` iff the ``.faiss`` loads as the new ``IndexIDMap2`` format;
72 ``False`` if it loads as a DIFFERENT (old) format; ``None`` if it could NOT
73 be read at all.
75 Used to tell an already-migrated index (IndexIDMap2, no sidecar) apart from
76 a phase-1 reindex-fallback leftover (an OLD-format base index whose ``.pkl``
77 was deleted without writing a sidecar). The caller quarantines ONLY on a
78 definite ``False``. A ``None`` (transient IO/permission error) must NOT
79 destructively quarantine + full-re-embed a possibly-healthy index — the
80 caller withholds the marker and retries next login instead. A genuinely
81 corrupt file that keeps failing is still caught by the normal index/search
82 load-failure path, which quarantines there.
83 """
84 from faiss import IndexIDMap2, read_index
86 try:
87 index = read_index(str(faiss_path))
88 except Exception:
89 return None
90 return isinstance(index, IndexIDMap2)
93def _load_uuid_to_id(
94 username: str, db_password: str, uuids: Iterable[str]
95) -> Dict[str, int]:
96 """Resolve ``DocumentChunk.embedding_id`` (uuid) -> ``DocumentChunk.id``
97 (int) for the given uuids, chunking the ``IN()`` list. uuids with no
98 matching row (deleted chunks) are simply absent from the result --
99 ``rekey_index_file`` drops those positions as orphans."""
100 uuid_list = list(uuids)
101 uuid_to_id: Dict[str, int] = {}
102 with get_user_db_session(username, db_password) as session:
103 for i in range(0, len(uuid_list), _IN_CLAUSE_CHUNK):
104 batch = uuid_list[i : i + _IN_CLAUSE_CHUNK]
105 rows = (
106 session.query(DocumentChunk.embedding_id, DocumentChunk.id)
107 .filter(DocumentChunk.embedding_id.in_(batch))
108 .all()
109 )
110 uuid_to_id.update(dict(rows))
111 return uuid_to_id
114def _resolve_index_params(rag_index: RAGIndex, settings) -> Dict:
115 """RAGIndex row values win; fall back to the current ``local_search_*``
116 defaults for legacy rows that predate these columns -- mirrors the
117 default resolution in ``rag_service_factory.get_rag_service``."""
118 return {
119 "dimension": rag_index.embedding_dimension,
120 "index_type": rag_index.index_type
121 or (settings.get_setting("local_search_index_type") or "flat"),
122 "metric": rag_index.distance_metric
123 or (settings.get_setting("local_search_distance_metric") or "cosine"),
124 "normalize": (
125 rag_index.normalize_vectors
126 if rag_index.normalize_vectors is not None
127 else settings.get_bool_setting("local_search_normalize_vectors")
128 ),
129 }
132def _persist_rekeyed_config(
133 username: str, db_password: str, rag_index_id: int, params: Dict
134) -> None:
135 """Pin the config the physical file was ACTUALLY built with onto the
136 RAGIndex row.
138 Legacy rows predate the index_type/distance_metric/normalize_vectors
139 columns, so ``_resolve_index_params`` fills them from the live
140 ``local_search_*`` settings and ``rekey_index_file`` bakes that choice into
141 the .faiss file. If the row is left NULL, every later access re-resolves the
142 SAME settings-fallback chain — so changing a global default afterward makes
143 the runtime reinterpret the file under a different index_type/metric/
144 normalize than it was built with. Write the resolved values back (only where
145 the row is still NULL — never override an explicit stored value)."""
146 with get_user_db_session(username, db_password) as session:
147 row = session.query(RAGIndex).filter_by(id=rag_index_id).first()
148 if row is None: 148 ↛ 149line 148 didn't jump to line 149 because the condition on line 148 was never true
149 return
150 changed = False
151 if row.index_type is None:
152 row.index_type = params["index_type"]
153 changed = True
154 if row.distance_metric is None:
155 row.distance_metric = params["metric"]
156 changed = True
157 if row.normalize_vectors is None:
158 row.normalize_vectors = params["normalize"]
159 changed = True
160 if changed:
161 session.commit()
164def _purge_redundant_pkl(faiss_path: Path, db_password: str) -> bool:
165 """Delete a legacy plaintext ``.pkl`` that has become REDUNDANT.
167 Once phase-1 has extracted the sidecar (or the index is already an
168 IndexIDMap2), the ``.pkl``'s chunk text lives only in the encrypted DB and
169 the id-map lives in the sidecar, so the ``.pkl`` is pure leftover plaintext.
170 Phase-1 deletes it, but that one-time unlink can fail (a directory the app
171 can't write, a read-only mount, an AV/EDR lock) AFTER the sidecar was
172 written — leaving plaintext on disk that no later step revisits once the
173 per-user completion marker is set. Callers invoke this at every phase-2
174 finalization point (re-key success, already-migrated skip) as a backstop.
176 Returns True if the ``.pkl`` is absent or was deleted, False if deletion
177 failed (the caller then withholds the completion marker so phase-2 retries
178 instead of gating the user out with plaintext still on disk). Deliberately
179 NOT called where a ``.pkl`` is still NEEDED (no sidecar + old-format index,
180 i.e. phase-1 hasn't converted it yet) — that case is handled separately.
181 """
182 pkl = faiss_path.with_suffix(".pkl")
183 if not pkl.exists():
184 return True
185 try:
186 pkl.unlink()
187 logger.info(f"Removed surviving legacy plaintext .pkl {pkl}")
188 return True
189 except OSError as exc:
190 safe_msg = redact_secrets(str(exc), db_password)
191 logger.warning(
192 f"Could not remove surviving legacy plaintext .pkl {pkl}; "
193 "withholding re-key completion so it is retried next login: "
194 f"{safe_msg}"
195 )
196 return False
199def _quarantine_and_reset(
200 username: str,
201 db_password: str,
202 rag_index: RAGIndex,
203 faiss_path: Path,
204 sidecar_path: Path,
205 lock,
206) -> None:
207 """Move the broken index pair aside and reset the collection's indexed
208 state so the normal indexing path rebuilds it from scratch.
210 Mirrors ``LibraryRAGService._quarantine_corrupt_index`` +
211 ``_reset_index_state_for_rebuild`` in spirit (rename-aside, never
212 delete; clear ``RagDocumentStatus`` + ``DocumentCollection.indexed`` so
213 the next index run does real work instead of skipping everything).
214 Kept as a free function here (rather than reusing those bound methods
215 directly) so this module doesn't have to construct a full
216 ``LibraryRAGService`` -- which would pull in an embedding provider --
217 just to quarantine a broken re-key.
219 Raises on rename failure so the caller can log it distinctly and, in
220 turn, withhold the per-user completion marker (see
221 ``rekey_user_indexes``) so this index is retried on the next login
222 instead of being silently abandoned behind a "done" marker.
223 """
224 # Reset DB state FIRST, then move the files aside. These two steps are not
225 # atomic, so ordering decides what a crash in between leaves behind:
226 # - DB-first (here): the collection is durably flagged "needs reindex"
227 # before its file is touched. A crash after the commit leaves a stray
228 # (still-original or renamed) file that the next index run simply
229 # rebuilds over — no data lost, no wrong state.
230 # - file-first (the reverse) would strand the collection as "indexed" with
231 # its backing index renamed away and no recoverable data until an admin
232 # intervenes — the exact permanent-loss failure this ordering avoids.
233 collection_id = rag_index.collection_name.removeprefix("collection_")
234 with get_user_db_session(username, db_password) as session:
235 idx = session.query(RAGIndex).filter_by(id=rag_index.id).first()
236 if idx: 236 ↛ 239line 236 didn't jump to line 239 because the condition on line 236 was always true
237 idx.chunk_count = 0
238 idx.total_documents = 0
239 session.query(RagDocumentStatus).filter_by(
240 rag_index_id=rag_index.id
241 ).delete()
242 # Only the CURRENT index backs the collection's live searches, so only
243 # its failure should force the whole collection to re-embed. Quarantining
244 # a STALE (is_current=False) row must NOT flip the collection's live
245 # DocumentCollection.indexed state -- the current index is still healthy
246 # and serving searches; clearing this row's RAGIndex + RagDocumentStatus
247 # above is enough. (rekey_user_indexes now re-keys stale rows too, so
248 # this guard is what keeps a stale-row failure from needlessly
249 # re-embedding the whole collection.)
250 if idx is not None and idx.is_current:
251 session.query(DocumentCollection).filter_by(
252 collection_id=collection_id, indexed=True
253 ).update({"indexed": False, "chunk_count": 0})
254 session.commit()
256 # Hold the per-(user, index_path) write lock for the renames so they don't
257 # race a concurrent search/index reading the same .faiss (which would see
258 # the file vanish mid-operation).
259 ns = time.time_ns()
260 with lock:
261 if faiss_path.exists(): 261 ↛ 262line 261 didn't jump to line 262 because the condition on line 261 was never true
262 faiss_path.rename(Path(f"{faiss_path}.corrupt-{ns}"))
263 if sidecar_path.exists(): 263 ↛ 264line 263 didn't jump to line 264 because the condition on line 263 was never true
264 sidecar_path.rename(Path(f"{sidecar_path}.corrupt-{ns}"))
265 # Also DELETE (not rename) any lingering legacy plaintext .pkl companion.
266 # Phase-1 normally removes it at startup, but if it couldn't then
267 # (unreadable/symlinked at that moment) or it reappeared, quarantining
268 # the .faiss must not leave the plaintext docstore on disk. Renaming
269 # aside like the .faiss would preserve the plaintext, so unlink it. Let
270 # an unlink failure PROPAGATE (like the renames above): swallowing it
271 # counted the quarantine as clean success and set the completion marker
272 # while plaintext survived on disk. The caller instead logs it and
273 # withholds the marker so phase-2 retries.
274 pkl = faiss_path.with_suffix(".pkl")
275 if pkl.exists(): 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true
276 pkl.unlink()
277 logger.warning(
278 f"Quarantined un-rekeyable RAG index {faiss_path} for collection "
279 f"{collection_id}; reset its indexed state so it rebuilds on next use."
280 )
283def rekey_user_indexes(username: str, db_password: str) -> Dict[str, int]:
284 """Re-key this user's legacy RAG indices (phase 2 of the cutover), once.
286 MUST be called only after ``username``'s DB connection is already in
287 ``db_manager.connections``. This function goes through
288 ``get_user_db_session`` / ``FileIntegrityManager``, both of which
289 re-enter ``db_manager.open_user_database`` on a thread-local
290 session-cache miss. Calling this from inside ``_open_user_database_cold``
291 BEFORE the connection is cached would deadlock: ``open_user_database``
292 serializes the cold-open behind a per-user ``threading.Lock`` (not
293 reentrant), which this same thread is already holding at that point.
295 Returns a stats dict (``checked``/``rekeyed``/``quarantined``/
296 ``skipped_no_sidecar``/``skipped_no_faiss``). Does not raise -- every
297 per-index failure is caught and quarantined; the caller wraps this call
298 anyway as defense in depth.
299 """
300 stats = {
301 "checked": 0,
302 "rekeyed": 0,
303 "quarantined": 0,
304 "skipped_no_sidecar": 0,
305 "skipped_no_faiss": 0,
306 }
308 with get_user_db_session(username, db_password) as session:
309 settings = get_settings_manager(db_session=session, username=username)
310 if settings.get_bool_setting(REKEY_MARKER_SETTING, False):
311 return stats # already done for this user -- fast no-op
312 # Re-key EVERY RAGIndex row phase-1 converted, not just the current one.
313 # A collection can have stale historical rows (is_current=False) from an
314 # earlier embedding-model switch, and those are NOT unreachable: reverting
315 # the embedding-model setting re-resolves a search to the stale row by
316 # index_hash (library_rag_service._get_or_create_rag_index looks the row
317 # up by hash, NOT is_current), and a still-old-format .faiss then fails to
318 # load and hard-fails search. Phase-1 already writes text-free sidecars
319 # for ALL of them, so phase-2 must re-key ALL of them. To keep a stale
320 # row's re-key FAILURE from collaterally wiping the healthy current
321 # index's indexed state, _quarantine_and_reset resets the collection-wide
322 # DocumentCollection.indexed flag ONLY for the current row (see there).
323 indexes: List[RAGIndex] = session.query(RAGIndex).all()
325 if not indexes:
326 settings.set_setting(REKEY_MARKER_SETTING, True)
327 return stats
329 # Lazy import: avoids a module-load cycle (this is a leaf import used
330 # only inside this function -- see rekey_index_file's docstring for the
331 # same pattern in legacy_cleanup.py).
332 from ..research_library.services.library_rag_service import (
333 _get_faiss_write_lock,
334 )
336 integrity_manager = None # built lazily -- only if a sidecar is found
337 # Only set the completion marker if every index was actually resolved
338 # (rekeyed / skipped / quarantined). If quarantining itself fails (e.g.
339 # disk full mid-rename), that index is left untouched and the marker is
340 # withheld so the NEXT login retries it, instead of silently abandoning
341 # it forever behind a "done" marker.
342 all_resolved = True
344 for rag_index in indexes:
345 stats["checked"] += 1
346 faiss_path = Path(rag_index.index_path)
347 sidecar_path = _sidecar_for(faiss_path)
348 # Per-(user, index_path) write lock, held for every file mutation
349 # (re-key persist, quarantine renames) so none race a concurrent
350 # search/index on the same .faiss.
351 lock = _get_faiss_write_lock(username, str(faiss_path))
353 if not sidecar_path.exists():
354 # No sidecar means EITHER already fully migrated (the .faiss is an
355 # IndexIDMap2) OR phase-1 hit its reindex-fallback: a corrupt/
356 # malformed legacy .pkl was deleted WITHOUT writing a sidecar,
357 # leaving the OLD-format .faiss stranded. The latter would otherwise
358 # be skipped forever while the DB still claims the collection is
359 # indexed, so its search silently breaks. Distinguish by format and
360 # quarantine + reset the un-migratable one so it rebuilds. (This
361 # runs once — the whole function is per-user marker-gated.)
362 fmt = _is_idmap2(faiss_path) if faiss_path.exists() else True
363 if fmt is None:
364 # Could NOT read the .faiss to determine its format (transient
365 # IO/permission). Do NOT quarantine a possibly-healthy index —
366 # withhold the marker and retry next login.
367 logger.warning(
368 f"Could not read {faiss_path} to check its format; will "
369 "retry next login (NOT quarantining a possibly-healthy "
370 "index)."
371 )
372 all_resolved = False
373 continue
374 if faiss_path.exists() and fmt is False:
375 # A no-sidecar old-format .faiss has TWO causes, and only one is
376 # the "phase-1 gave up" (reindex-fallback) case that warrants a
377 # destructive quarantine. If the legacy plaintext .pkl companion
378 # STILL EXISTS, phase-1 simply hasn't converted this file yet
379 # (it ran before this collection existed / crashed early / a
380 # sibling process's phase-1 hasn't reached it) — its sidecar is
381 # still to come. Quarantining now would force a needless full
382 # re-embed instead of the cheap rekey phase-1 will enable, and it
383 # would delete an intact docstore out from under phase-1. Withhold
384 # the completion marker and retry next login, by which point
385 # phase-1 (which runs every startup) will have written the
386 # sidecar and removed the .pkl.
387 if faiss_path.with_suffix(".pkl").exists(): 387 ↛ 395line 387 didn't jump to line 395 because the condition on line 387 was always true
388 logger.warning(
389 f"Old-format index {faiss_path} still has its legacy "
390 ".pkl and no sidecar — phase-1 has not converted it yet; "
391 "will retry next login (NOT quarantining)."
392 )
393 all_resolved = False
394 continue
395 logger.warning(
396 f"Un-migratable old-format index {faiss_path} has no "
397 "sidecar (phase-1 reindex-fallback); quarantining so its "
398 "collection rebuilds."
399 )
400 try:
401 _quarantine_and_reset(
402 username,
403 db_password,
404 rag_index,
405 faiss_path,
406 sidecar_path,
407 lock,
408 )
409 stats["quarantined"] += 1
410 except Exception as exc:
411 # db_password is a live parameter of this function; a
412 # rendered traceback (loguru diagnose=True) would dump it.
413 # Drop the traceback and redact str(exc). See encrypted_db.
414 safe_msg = redact_secrets(str(exc), db_password)
415 logger.warning(
416 f"Failed to quarantine {faiss_path}; will retry next "
417 f"login: {safe_msg}"
418 )
419 all_resolved = False
420 continue
421 # Already migrated (IndexIDMap2, no sidecar). Backstop: purge any
422 # legacy plaintext .pkl that survived phase-1 here too — this is the
423 # retry landing spot after a re-key whose Phase-B .pkl unlink failed
424 # (sidecar already gone), so without it the marker would be set with
425 # plaintext still on disk.
426 if not _purge_redundant_pkl(faiss_path, db_password): 426 ↛ 427line 426 didn't jump to line 427 because the condition on line 426 was never true
427 all_resolved = False
428 stats["skipped_no_sidecar"] += 1
429 continue
430 if not faiss_path.exists(): 430 ↛ 441line 430 didn't jump to line 441 because the condition on line 430 was never true
431 # Orphaned sidecar (index file itself is already gone) -- there
432 # is nothing to re-key. Purge any legacy plaintext .pkl that
433 # survived next to the now-gone .faiss (a phase-1 .pkl-unlink
434 # failure, or a partial _quarantine_and_reset that renamed the
435 # .faiss aside before reaching its own pkl.unlink, can leave one
436 # here) BEFORE dropping the sidecar. Every other finalization branch
437 # backstops the plaintext .pkl this way; without it this branch is
438 # the one path that can set the per-user completion marker while an
439 # encryption-defeating plaintext .pkl still sits on disk. Withhold
440 # the marker if EITHER cleanup fails so phase-2 retries next login.
441 stats["skipped_no_faiss"] += 1
442 if not _purge_redundant_pkl(faiss_path, db_password):
443 all_resolved = False
444 try:
445 sidecar_path.unlink(missing_ok=True)
446 except OSError:
447 logger.warning(
448 f"Could not remove orphaned sidecar {sidecar_path}; "
449 "will retry next login"
450 )
451 all_resolved = False
452 continue
454 # Resolve + PIN the index config onto the row BEFORE building the file,
455 # so the row records what the file is built with even across a
456 # crash/retry. Pinning AFTER a successful build would break the retry
457 # path: if settings change between a crashed attempt and the retry, the
458 # retry re-resolves DIFFERENT params, hits rekey_index_file's
459 # already-migrated short-circuit (which only checks dim + ids), and pins
460 # a config that disagrees with the physical file. Pinning FIRST makes the
461 # row the source of truth that _resolve_index_params reads on retry (row
462 # wins over settings), so the resolved params match the file at every
463 # crash point. A transient failure here must NOT quarantine a healthy
464 # legacy file — withhold the marker and retry next login.
465 try:
466 params = _resolve_index_params(rag_index, settings)
467 # SQLAlchemy's legacy Column() declarative style types instance
468 # attribute access as Column[int] rather than int; cast() is a
469 # no-op at runtime.
470 _persist_rekeyed_config(
471 username, db_password, cast(int, rag_index.id), params
472 )
473 except Exception as exc:
474 safe_msg = redact_secrets(str(exc), db_password)
475 logger.warning(
476 f"Failed to pin index config for RAG index {rag_index.id} "
477 f"({faiss_path}); will retry next login: {safe_msg}"
478 )
479 all_resolved = False
480 continue
482 # Resolve the sidecar map + uuid->DocumentChunk.id (a DB read). A failure
483 # HERE is transient and unrelated to the index file's health — a
484 # 'database is locked' timeout under concurrent writes, or a momentary
485 # OSError reading the sidecar. It must NOT quarantine a perfectly healthy
486 # legacy index (which would force a needless full re-embed); withhold the
487 # marker and retry next login instead.
488 try:
489 pos_to_uuid = json.loads(sidecar_path.read_text(encoding="utf-8"))
490 uuid_to_id = _load_uuid_to_id(
491 username, db_password, set(pos_to_uuid.values())
492 )
493 except Exception as exc:
494 safe_msg = redact_secrets(str(exc), db_password)
495 logger.warning(
496 f"Could not resolve sidecar/DB ids for RAG index "
497 f"{rag_index.id} ({faiss_path}); will retry next login "
498 f"(NOT quarantining — the index file is untouched): {safe_msg}"
499 )
500 all_resolved = False
501 continue
503 # Phase A -- re-key. A failure HERE is a genuinely un-rekeyable index
504 # (corrupt/foreign format, dimension mismatch, duplicate-id sidecar):
505 # quarantine + reset so the collection rebuilds from the DB.
506 try:
507 with lock:
508 rekey_index_file(faiss_path, sidecar_path, uuid_to_id, **params)
509 except Exception as exc:
510 safe_msg = redact_secrets(str(exc), db_password)
511 logger.warning(
512 f"Re-key failed for RAG index {rag_index.id} "
513 f"({faiss_path}); quarantining -- collection will reindex: "
514 f"{safe_msg}"
515 )
516 try:
517 _quarantine_and_reset(
518 username,
519 db_password,
520 rag_index,
521 faiss_path,
522 sidecar_path,
523 lock,
524 )
525 stats["quarantined"] += 1
526 except Exception as exc:
527 safe_msg = redact_secrets(str(exc), db_password)
528 logger.warning(
529 f"Failed to quarantine {faiss_path} after re-key "
530 f"failure -- leaving files as-is; will retry next login: "
531 f"{safe_msg}"
532 )
533 all_resolved = False
534 continue
536 # Phase B -- the re-key SUCCEEDED (byte-verified new IndexIDMap2).
537 # Record the new file's integrity, THEN delete the sidecar (the
538 # completion signal). A failure here must NOT quarantine a perfectly
539 # good index (the bug this split fixes): leave the sidecar so the next
540 # login retries -- rekey_index_file's IndexIDMap2 guard makes that a
541 # clean no-op re-key -- and withhold the completion marker.
542 try:
543 with lock:
544 if integrity_manager is None:
545 integrity_manager = FileIntegrityManager(
546 username, db_password
547 )
548 integrity_manager.register_verifier(FAISSIndexVerifier())
549 integrity_manager.record_file(
550 faiss_path,
551 related_entity_type="rag_index",
552 # cast(): rag_index.id is Column[int] under the legacy
553 # Column() declarative style; a no-op at runtime.
554 related_entity_id=cast(int, rag_index.id),
555 )
556 # Config was already pinned before the build (see above), so
557 # Phase B only records integrity and drops the sidecar signal.
558 sidecar_path.unlink(missing_ok=True)
559 # Backstop: if phase-1 wrote the sidecar but its .pkl unlink
560 # failed, the plaintext .pkl is still on disk. The re-key just
561 # succeeded from the sidecar and the text is in the DB, so purge
562 # the now-redundant .pkl; withhold the marker if it can't be
563 # removed so this index is retried rather than gated out with
564 # plaintext surviving.
565 if not _purge_redundant_pkl(faiss_path, db_password): 565 ↛ 566line 565 didn't jump to line 566 because the condition on line 565 was never true
566 all_resolved = False
567 stats["rekeyed"] += 1
568 except Exception as exc:
569 safe_msg = redact_secrets(str(exc), db_password)
570 logger.warning(
571 f"Re-key of {faiss_path} succeeded but finalization "
572 "(integrity record / sidecar removal) failed; leaving sidecar "
573 f"to retry next login: {safe_msg}"
574 )
575 all_resolved = False
577 if all_resolved:
578 settings.set_setting(REKEY_MARKER_SETTING, True)
579 logger.info(
580 f"RAG index re-key for user {username}: "
581 f"checked={stats['checked']} rekeyed={stats['rekeyed']} "
582 f"quarantined={stats['quarantined']} "
583 f"skipped_no_sidecar={stats['skipped_no_sidecar']} "
584 f"skipped_no_faiss={stats['skipped_no_faiss']}"
585 + ("" if all_resolved else " (incomplete -- will retry next login)")
586 )
587 return stats