Coverage for src/local_deep_research/vector_stores/base.py: 100%
29 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-20 07:11 +0000
1"""Abstract base class for vector stores.
3Mirrors the embeddings/LLM provider pattern (``BaseEmbeddingProvider``,
4``BaseLLM`` + factory): a small typed interface plus a registry so a future
5release can add backends (Qdrant, pgvector, Milvus, ...) selectable by a
6setting. **Only FAISS is implemented today** — there is intentionally no
7settings selector wired yet.
9Scope / honest boundary
10-----------------------
11This abstraction covers the *vector-storage operations* — create / add /
12search / delete / reconstruct / persist / count. It deliberately does **not**
13cover the per-user file paths, on-disk integrity checksums, or the per-index
14write-lock + reload/merge concurrency model that :class:`LibraryRAGService`
15layers on top. Those are *local-file* concerns (see :attr:`is_local_file`): a
16future server-backed store (e.g. Qdrant) would set ``is_local_file = False``,
17no-op :meth:`persist` / :meth:`load`, and the service layer would skip the file
18lock + integrity machinery for it. So this base class makes adding a backend
19*easier* (a clean, verified query surface), not *free* — a real server backend
20still requires refactoring the file/lock/integrity layer in the service.
22Identity model
23--------------
24Vectors are keyed by an application-supplied **int64 id**. In this codebase
25that id is ``DocumentChunk.id`` (the encrypted-DB primary key); the store holds
26only vectors + ids, and all text/metadata is rehydrated from the DB by id.
28============================================================================
29SECURITY INVARIANT — NEVER pass document text to a vector store
30============================================================================
31Every method here accepts ONLY an integer id and its embedding vector.
32Callers MUST NOT pass document/chunk text (or any other user content) into a
33vector-store method, and implementations MUST NOT persist such text.
35The per-user database is encrypted; a vector index is not (it is a local file
36for FAISS, or an external service for a future backend). Keeping user content
37out of the vector store keeps the encrypted database the sole home of that
38content. Because the interface has no text parameter, the protection is
39STRUCTURAL rather than a matter of discipline — no backend (FAISS today, or a
40swapped-in Qdrant/pgvector tomorrow) can receive or store the text. The
41authoritative text lives solely in the encrypted DB
42(``DocumentChunk.chunk_text``), keyed by the same int id; retrieval rehydrates
43snippets from there by id (see ``LibraryRAGService`` / the search engines).
44============================================================================
45"""
47from abc import ABC, abstractmethod
48from pathlib import Path
49from typing import List, Optional, Sequence, Tuple
51import numpy as np
54class BaseVectorStore(ABC):
55 """Abstract interface for a per-collection vector index.
57 An instance wraps a single collection's index (stateful, in-memory, backed
58 by a persisted file for local-file stores). Construct via :meth:`create`
59 (new, empty) or :meth:`load` (from disk).
60 """
62 # Override in subclasses.
63 provider_key: str = "base" # unique id; matches the (future) setting value
64 provider_name: str = "Base" # display name for logs/UI
65 # True when the store persists to a local file this process owns end to end
66 # — i.e. the service's file write-lock + integrity checksum + reload/merge
67 # model applies. A server-backed store sets this False.
68 is_local_file: bool = True
69 # True when the store can invert id -> vector (needed for the in-place
70 # format migration, which re-keys vectors without re-embedding).
71 supports_reconstruct: bool = True
72 # Embedding dimension this instance was constructed/loaded for. Concrete
73 # subclasses set this in __init__/create()/load(); declared here (no
74 # default — there is no sensible universal value) so callers like the
75 # facade can read it via the abstract interface.
76 dimension: int
78 # ------------------------------------------------------------------ #
79 # Construction
80 # ------------------------------------------------------------------ #
81 @classmethod
82 @abstractmethod
83 def create(
84 cls,
85 *,
86 dimension: int,
87 index_type: str,
88 metric: str,
89 normalize: bool,
90 **kwargs,
91 ) -> "BaseVectorStore":
92 """Create a new, empty store for vectors of ``dimension``.
94 Args:
95 dimension: Embedding dimension.
96 index_type: Backend index family (e.g. ``"flat"``, ``"hnsw"``).
97 metric: Distance metric (``"l2"``, ``"cosine"``, ``"dot_product"``).
98 normalize: Whether query/doc vectors are L2-normalized before use
99 (cosine similarity via inner product on normalized vectors).
100 """
102 @classmethod
103 @abstractmethod
104 def load(
105 cls,
106 path: Path,
107 *,
108 dimension: int,
109 index_type: str,
110 metric: str,
111 normalize: bool,
112 **kwargs,
113 ) -> "BaseVectorStore":
114 """Load a persisted store from ``path`` (local-file stores only).
116 ``dimension`` / ``index_type`` / ``metric`` / ``normalize`` describe how
117 the index was built (some backends persist this, some do not); callers
118 pass the values from settings so search-time normalization matches
119 build-time normalization.
120 """
122 # ------------------------------------------------------------------ #
123 # Vector operations
124 # ------------------------------------------------------------------ #
125 @abstractmethod
126 def add(self, ids: List[int], vectors: np.ndarray) -> None:
127 """Add vectors under the given int64 ids (aligned 1:1, same length).
129 ids + vectors ONLY — never text (see the module-level "SECURITY
130 INVARIANT" block). The text stays in the encrypted DB.
131 """
133 @abstractmethod
134 def search(
135 self, query_vector: np.ndarray, k: int
136 ) -> List[Tuple[int, float]]:
137 """Return up to ``k`` ``(id, distance)`` pairs, nearest first.
139 ``distance`` is the backend's raw score (L2 distance or inner product);
140 callers map it to a relevance score. Absent/empty slots are filtered.
141 """
143 @abstractmethod
144 def delete(self, ids: List[int]) -> int:
145 """Remove the given ids. Returns the number actually removed.
147 May raise for index families that do not support removal (e.g. FAISS
148 HNSW) — callers handle that the same way they did pre-abstraction.
149 """
151 @abstractmethod
152 def live_ids(self) -> List[int]:
153 """Return every id currently stored (the authoritative membership)."""
155 @abstractmethod
156 def count(self) -> int:
157 """Return the number of vectors currently stored."""
159 @abstractmethod
160 def apply(
161 self,
162 *,
163 add_ids: List[int],
164 add_vectors: Optional[np.ndarray],
165 remove_ids: Sequence[int] = (),
166 dedup: bool = True,
167 ) -> dict:
168 """Durably apply a batch of removals + additions as one atomic unit.
170 This is the single write primitive. The caller computes *which* ids to
171 remove and add (from its own authoritative source — for LDR, the
172 encrypted DB); the store applies them and persists so that a concurrent
173 writer to the same logical index cannot lose either party's changes.
174 *How* that atomicity/isolation is achieved is backend-specific (a
175 local-file backend takes a per-index lock and reload-merges; a
176 server-backed store issues upserts/deletes) and is not part of this
177 contract.
179 Contract: removals are applied before additions, and ``dedup`` skips
180 additions whose id is already present after removal — so a "replace" is
181 expressed as ``remove_ids`` + ``add_ids`` of the same id. On failure the
182 store must not be left durably half-applied.
184 Returns a stats dict (e.g. ``{"added": n, "removed": m}``). ids +
185 vectors ONLY — never text (see the module "SECURITY INVARIANT").
186 """
188 # ------------------------------------------------------------------ #
189 # Optional (local-file / reconstructable stores)
190 # ------------------------------------------------------------------ #
191 def reconstruct(self, id: int) -> Optional[np.ndarray]:
192 """Return the stored vector for ``id`` (migration helper).
194 Only meaningful when :attr:`supports_reconstruct` is True.
195 """
196 raise NotImplementedError(
197 f"{type(self).__name__} does not support reconstruct()"
198 )
200 def persist(self, path: Path) -> None:
201 """Durably write the store to ``path`` (local-file stores only)."""
202 raise NotImplementedError(
203 f"{type(self).__name__} does not support persist()"
204 )