Coverage for src/local_deep_research/vector_stores/base.py: 92%
36 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""Abstract base class for vector stores.
3Mirrors the embeddings/LLM provider pattern (``BaseEmbeddingProvider``,
4``BaseLLM`` + factory): a small typed interface plus a registry so a future
5release can add backends (Qdrant, pgvector, Milvus, ...) selectable by a
6setting. **Only FAISS is implemented today** — there is intentionally no
7settings selector wired yet.
9Scope / honest boundary
10-----------------------
11This abstraction covers the *vector-storage operations* — create / add /
12search / delete / reconstruct / persist / count. Local-file constructors accept
13injected persistence resources, but this interface deliberately does **not**
14prescribe their per-user paths, integrity checksums, or reload/merge locking
15model that :class:`LibraryRAGService` layers on top. Those are *local-file*
16concerns (see :attr:`is_local_file`): a future server-backed store (e.g.
17Qdrant) would set ``is_local_file = False``, no-op :meth:`persist` /
18:meth:`load`, and the service layer would skip the file lock + integrity
19machinery for it. So this base class makes adding a backend *easier* (a clean,
20verified query surface), not *free* — a real server backend still requires
21refactoring the file/lock/integrity layer in the service.
23Identity model
24--------------
25Vectors are keyed by an application-supplied **int64 id**. In this codebase
26that id is ``DocumentChunk.id`` (the encrypted-DB primary key); the store holds
27only vectors + ids, and all text/metadata is rehydrated from the DB by id.
29============================================================================
30SECURITY INVARIANT — NEVER pass document text to a vector store
31============================================================================
32Every method here accepts ONLY an integer id and its embedding vector.
33Callers MUST NOT pass document/chunk text (or any other user content) into a
34vector-store method, and implementations MUST NOT persist such text.
36The per-user database is encrypted; a vector index is not (it is a local file
37for FAISS, or an external service for a future backend). Keeping user content
38out of the vector store keeps the encrypted database the sole home of that
39content. Because the interface has no text parameter, the protection is
40STRUCTURAL rather than a matter of discipline — no backend (FAISS today, or a
41swapped-in Qdrant/pgvector tomorrow) can receive or store the text. The
42authoritative text lives solely in the encrypted DB
43(``DocumentChunk.chunk_text``), keyed by the same int id; retrieval rehydrates
44snippets from there by id (see ``LibraryRAGService`` / the search engines).
45============================================================================
46"""
48from abc import ABC, abstractmethod
49from pathlib import Path
50from types import TracebackType
51from typing import Callable, List, Optional, Protocol, Self, Sequence, Tuple
53import numpy as np
56IntegrityRecord = Callable[[Path], None]
57IntegrityVerify = Callable[[Path], Tuple[bool, Optional[str]]]
60class WriteLock(Protocol):
61 """Minimal lock capability used to guard a vector store's file writes.
63 Implementations MUST be reentrant. Write paths nest acquisition on a
64 single thread — the service layer holds the lock across a document
65 operation while the store's own reload/apply/persist choreography
66 re-acquires it — so a plain ``threading.Lock`` satisfies this Protocol
67 structurally but self-deadlocks at runtime. Use ``threading.RLock`` (or
68 a wrapper around one, as the production tracked lock is).
69 """
71 def __enter__(self) -> bool | Self: ... 71 ↛ anywhereline 71 didn't jump anywhere: it always raised an exception.
73 def __exit__( 73 ↛ exitline 73 didn't return from function '__exit__' because
74 self,
75 exception_type: type[BaseException] | None,
76 exception: BaseException | None,
77 traceback: TracebackType | None,
78 ) -> None: ...
81class BaseVectorStore(ABC):
82 """Abstract interface for a per-collection vector index.
84 An instance wraps a single collection's index (stateful, in-memory, backed
85 by a persisted file for local-file stores). Construct via :meth:`create`
86 (new, empty) or :meth:`load` (from disk).
87 """
89 # Override in subclasses.
90 provider_key: str = "base" # unique id; matches the (future) setting value
91 provider_name: str = "Base" # display name for logs/UI
92 # True when the store persists to a local file this process owns end to end
93 # — i.e. the service's file write-lock + integrity checksum + reload/merge
94 # model applies. A server-backed store sets this False.
95 is_local_file: bool = True
96 # Settings key holding this store's endpoint URI, for a store that is NOT
97 # a local file. The egress policy reads it exactly as it reads a search
98 # engine's ``url_setting`` (``security/egress/policy.py``): a
99 # server-backed store whose configured host resolves to a PUBLIC address
100 # ships the collection's embeddings and every query off the machine, so
101 # the library/collection engines fail up to exposing/public and
102 # PRIVATE_ONLY denies them. A non-local-file store that declares no key
103 # cannot be shown to be on-box and is therefore treated as remote —
104 # declaring it is how a backend proves the opposite. Unused for a
105 # local-file store (FAISS), which is contained by construction.
106 uri_setting: Optional[str] = None
107 # True when the store can invert id -> vector (needed for the in-place
108 # format migration, which re-keys vectors without re-embedding).
109 supports_reconstruct: bool = True
110 # Embedding dimension this instance was constructed/loaded for. Concrete
111 # subclasses set this in __init__/create()/load(); declared here (no
112 # default — there is no sensible universal value) so callers like the
113 # facade can read it via the abstract interface.
114 dimension: int
116 # ------------------------------------------------------------------ #
117 # Construction
118 # ------------------------------------------------------------------ #
119 @classmethod
120 @abstractmethod
121 def create(
122 cls,
123 *,
124 dimension: int,
125 index_type: str,
126 metric: str,
127 normalize: bool,
128 path: Optional[Path] = None,
129 lock: Optional[WriteLock] = None,
130 integrity_record: Optional[IntegrityRecord] = None,
131 integrity_verify: Optional[IntegrityVerify] = None,
132 ) -> "BaseVectorStore":
133 """Create a new, empty store for vectors of ``dimension``.
135 Args:
136 dimension: Embedding dimension.
137 index_type: Backend index family (e.g. ``"flat"``, ``"hnsw"``).
138 metric: Distance metric (``"l2"``, ``"cosine"``, ``"dot_product"``).
139 normalize: Whether query/doc vectors are L2-normalized before use
140 (cosine similarity via inner product on normalized vectors).
141 path: Optional local persistence path.
142 lock: Optional reentrant lock for local persistence (see
143 :class:`WriteLock`).
144 integrity_record: Optional persisted-file integrity callback.
145 integrity_verify: Optional persisted-file integrity check.
146 """
148 @classmethod
149 @abstractmethod
150 def load(
151 cls,
152 path: Path,
153 *,
154 dimension: int,
155 index_type: str,
156 metric: str,
157 normalize: bool,
158 lock: Optional[WriteLock] = None,
159 integrity_record: Optional[IntegrityRecord] = None,
160 integrity_verify: Optional[IntegrityVerify] = None,
161 ) -> "BaseVectorStore":
162 """Load a persisted store from ``path`` (local-file stores only).
164 ``dimension`` / ``index_type`` / ``metric`` / ``normalize`` describe how
165 the index was built (some backends persist this, some do not); callers
166 pass the values from settings so search-time normalization matches
167 build-time normalization. ``lock`` / ``integrity_record`` /
168 ``integrity_verify`` are the same injected persistence resources as on
169 :meth:`create`.
170 """
172 # ------------------------------------------------------------------ #
173 # Vector operations
174 # ------------------------------------------------------------------ #
175 @abstractmethod
176 def add(self, ids: List[int], vectors: np.ndarray) -> None:
177 """Add vectors under the given int64 ids (aligned 1:1, same length).
179 ids + vectors ONLY — never text (see the module-level "SECURITY
180 INVARIANT" block). The text stays in the encrypted DB.
181 """
183 @abstractmethod
184 def search(
185 self, query_vector: np.ndarray, k: int
186 ) -> List[Tuple[int, float]]:
187 """Return up to ``k`` ``(id, distance)`` pairs, nearest first.
189 ``distance`` is the backend's raw score (L2 distance or inner product);
190 callers map it to a relevance score. Absent/empty slots are filtered.
191 """
193 @abstractmethod
194 def delete(self, ids: List[int]) -> int:
195 """Remove the given ids. Returns the number actually removed.
197 May raise for index families that do not support removal (e.g. FAISS
198 HNSW) — callers handle that the same way they did pre-abstraction.
199 """
201 @abstractmethod
202 def live_ids(self) -> List[int]:
203 """Return every id currently stored (the authoritative membership)."""
205 @abstractmethod
206 def count(self) -> int:
207 """Return the number of vectors currently stored."""
209 @abstractmethod
210 def apply(
211 self,
212 *,
213 add_ids: List[int],
214 add_vectors: Optional[np.ndarray],
215 remove_ids: Sequence[int] = (),
216 dedup: bool = True,
217 ) -> dict:
218 """Durably apply a batch of removals + additions as one atomic unit.
220 This is the single write primitive. The caller computes *which* ids to
221 remove and add (from its own authoritative source — for LDR, the
222 encrypted DB); the store applies them and persists so that a concurrent
223 writer to the same logical index cannot lose either party's changes.
224 *How* that atomicity/isolation is achieved is backend-specific (a
225 local-file backend takes a per-index lock and reload-merges; a
226 server-backed store issues upserts/deletes) and is not part of this
227 contract.
229 Contract: removals are applied before additions. A "replace" is
230 expressed as ``remove_ids`` + ``add_ids`` of the same id.
232 ``dedup`` de-duplicates ids WITHIN one call (a repeated id in
233 ``add_ids`` is written once). It does NOT mean "skip an add whose id
234 is already live in the store" — an already-live ``add_id`` MUST be
235 replaced, and the NEW vector must win.
237 That is not a nicety. An ``add_id`` that is already live is either an
238 idempotent re-add or — critically — a REUSED ``DocumentChunk.id``
239 whose prior row was rolled back: SQLite recycles AUTOINCREMENT ids on
240 ROLLBACK, so a failed ``index()`` commit can leave a stale orphan
241 vector under an id that a later, unrelated chunk then reuses. Keeping
242 the old vector makes search rank on one document's embedding and then
243 rehydrate a DIFFERENT document's text by id. Nothing downstream
244 detects it: a reconciler that checks ``live_ids()`` membership sees a
245 healthy index, because the skipped add left the id live.
247 On failure the store must not be left durably half-applied.
249 Returns a stats dict (e.g. ``{"added": n, "removed": m}``). ids +
250 vectors ONLY — never text (see the module "SECURITY INVARIANT").
251 """
253 # ------------------------------------------------------------------ #
254 # Optional (local-file / reconstructable stores)
255 # ------------------------------------------------------------------ #
256 def reconstruct(self, id: int) -> Optional[np.ndarray]:
257 """Return the stored vector for ``id`` (migration helper).
259 Only meaningful when :attr:`supports_reconstruct` is True.
260 """
261 raise NotImplementedError(
262 f"{type(self).__name__} does not support reconstruct()"
263 )
265 def persist(self, path: Path) -> None:
266 """Durably write the store to ``path`` (local-file stores only)."""
267 raise NotImplementedError(
268 f"{type(self).__name__} does not support persist()"
269 )