Coverage for src/local_deep_research/web/routers/unified_search.py: 98%
130 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Unified Search Routes — "Search everything" (FastAPI).
4One search surface across every entity stored as a ``Document`` row
5(notes, research reports, library downloads, user uploads):
7* GET /library/search/ — the Search page
8* GET /library/search/api/keyword — ILIKE over the listed name + text_content
9* GET /library/search/api/semantic — FAISS search over the system
10 collections (default_library, research_history, notes)
12The two API legs return the SAME result-row contract so the page JS can
13merge them mode-agnostically::
15 {id, title, content_preview, source_type, url, similarity?}
17``url`` is computed server-side (see ``_document_url``) so every future
18consumer links results consistently.
20Ported from the Flask ``web/routes/unified_search_routes.py`` blueprint.
21"""
23import math
25from fastapi import APIRouter, Depends, Request
26from fastapi.responses import JSONResponse
28from ...research_library.utils import escape_like, handle_api_error
29from ..dependencies.auth import require_auth
30from ..dependencies.rate_limit import limiter, _user_key
31from ..template_config import templates
33# MAX_SEARCH_LEN (the user-query cap into ILIKE + the embedding call) is
34# shared with notes_routes via _search_constants so the two route modules
35# can't diverge.
36from ..routes._search_constants import MAX_SEARCH_LEN
37from typing import Annotated
39# Keyword-leg preview length: column-projected ``substr`` so the full
40# text_content (which can be megabytes for extracted PDFs) is never
41# hydrated into the response.
42PREVIEW_LEN = 300
43# Lead-in shown before the first content match when the preview window is
44# anchored on it (see keyword_search).
45PREVIEW_CONTEXT = 60
46DEFAULT_LIMIT = 20
47MAX_LIMIT = 50
48DEFAULT_MIN_SIMILARITY = 0.25
50# Minimum query length, matching the page JS's UNIFIED_SEARCH_MIN_QUERY:
51# below 2 chars the keyword leg is an expensive full-content scan for a
52# near-meaningless match set, and the client never sends one anyway.
53MIN_SEARCH_LEN = 2
55# System collections always searched by the semantic leg, keyed by
56# ``Collection.collection_type``. These are lazily created elsewhere
57# (library init / first note), so a missing one is SKIPPED rather than
58# created — a read endpoint must not have write side effects. User
59# collections join them whenever they have a current RAG index (see
60# semantic_search) so the two legs cover the same corpus for anything
61# the user actually indexed.
62SEMANTIC_COLLECTION_TYPES = ("default_library", "research_history", "notes")
64# Per-user search budget — same shape as the notes_search bucket (cheap
65# DB/FAISS lookups typed through a debounced search box).
66_unified_search_limit = limiter.shared_limit(
67 "60 per minute", scope="unified_search", key_func=_user_key
68)
70router = APIRouter(prefix="/library/search", tags=["unified-search"])
73def _document_url(source_type_name, document_id, research_id):
74 """Canonical link for a search hit, by source type.
76 * ``note`` → the note editor.
77 * ``research_report`` with a ``research_id`` → the research results
78 page (the report Document is a search-index copy of that run).
79 * everything else (uploads, downloads, report copies whose run is
80 gone) → the library document details page.
81 """
82 if source_type_name == "note":
83 return f"/notes/{document_id}"
84 if source_type_name == "research_report" and research_id:
85 return f"/results/{research_id}"
86 return f"/library/document/{document_id}"
89def _validated_query_and_limit(request: Request):
90 """Parse/validate the shared ``q`` and ``limit`` query params.
92 Returns ``(query, limit, error_response)`` where ``error_response``
93 is a ready ``JSONResponse`` when validation failed (and the other
94 fields are None).
95 """
96 q = request.query_params.get("q", "")
97 if len(q) > MAX_SEARCH_LEN:
98 return (
99 None,
100 None,
101 JSONResponse(
102 {
103 "success": False,
104 "error": f"q exceeds maximum length ({MAX_SEARCH_LEN} chars)",
105 },
106 status_code=400,
107 ),
108 )
109 q = q.strip()
110 if not q:
111 return (
112 None,
113 None,
114 JSONResponse(
115 {"success": False, "error": "Query is required"},
116 status_code=400,
117 ),
118 )
119 if len(q) < MIN_SEARCH_LEN:
120 return (
121 None,
122 None,
123 JSONResponse(
124 {
125 "success": False,
126 "error": (
127 f"Query must be at least {MIN_SEARCH_LEN} characters"
128 ),
129 },
130 status_code=400,
131 ),
132 )
133 try:
134 limit = max(
135 1,
136 min(
137 int(request.query_params.get("limit", DEFAULT_LIMIT)), MAX_LIMIT
138 ),
139 )
140 except (ValueError, TypeError):
141 return (
142 None,
143 None,
144 JSONResponse(
145 {"success": False, "error": "Invalid limit"}, status_code=400
146 ),
147 )
148 return q, limit, None
151# ============================================================================
152# Page Route
153# ============================================================================
156@router.get("/")
157def unified_search_page(
158 request: Request, username: Annotated[str, Depends(require_auth)]
159):
160 """Render the unified Search page."""
161 return templates.TemplateResponse(
162 request=request,
163 name="pages/unified_search.html",
164 context={"active_page": "unified-search"},
165 )
168# ============================================================================
169# API Routes
170# ============================================================================
173@router.get("/api/keyword")
174@_unified_search_limit
175def keyword_search(
176 request: Request, username: Annotated[str, Depends(require_auth)]
177):
178 """Keyword leg: ILIKE over the listed name + text_content, ALL source types.
180 The listed name is ``title or filename`` (``display_title`` below), so
181 the filename is searched only when the title is NULL or empty, as for
182 an uploaded document. A titled document is not matched by its stored
183 filename. ILIKE folds case for ASCII letters only.
185 Query params:
186 q: Search query (required, capped at MAX_SEARCH_LEN)
187 limit: Maximum results (default 20, clamped 1..50)
189 Column-projected (id, title, filename, 300-char preview, source-type name,
190 updated_at, research_id) — never hydrates full text_content.
191 """
192 try:
193 q, limit, error = _validated_query_and_limit(request)
194 if error:
195 return error
197 from sqlalchemy import and_, case, func, or_
199 from ...database.models.library import Document, SourceType
200 from ...database.session_context import get_user_db_session
202 pattern = f"%{escape_like(q)}%"
203 # Center the preview on the first content match instead of always
204 # slicing the head: for a content hit deep in a long document, the
205 # first 300 chars usually don't contain the matched text at all,
206 # so the card gave no clue why the document matched. instr() takes
207 # the raw query (wildcard escaping only applies to LIKE), folded
208 # with lower() on both sides exactly like ilike; 0 = name-only
209 # match → head preview. PREVIEW_CONTEXT chars of lead-in keep the
210 # match readable in situ.
211 match_pos = func.instr(func.lower(Document.text_content), func.lower(q))
212 preview_start = case(
213 (match_pos > PREVIEW_CONTEXT, match_pos - PREVIEW_CONTEXT),
214 else_=1,
215 )
216 with get_user_db_session(username) as db:
217 rows = (
218 db.query(
219 Document.id,
220 Document.title,
221 Document.filename,
222 func.substr(
223 Document.text_content, preview_start, PREVIEW_LEN
224 ),
225 SourceType.name,
226 Document.updated_at,
227 Document.research_id,
228 match_pos,
229 )
230 .join(SourceType, Document.source_type_id == SourceType.id)
231 .filter(
232 # Only surface completed documents, matching every
233 # library listing/search path (library_service filters
234 # status == 'completed' too). Without this, in-progress
235 # or failed downloads — partial/empty text, or a stale
236 # title — appear as hits that link to a document page
237 # with no usable content.
238 Document.status == "completed",
239 or_(
240 Document.title.ilike(pattern, escape="\\"),
241 # Results are listed as ``title or filename``, so
242 # the filename is searched exactly where it is
243 # the listed name.
244 and_(
245 or_(
246 Document.title.is_(None),
247 Document.title == "",
248 ),
249 Document.filename.ilike(pattern, escape="\\"),
250 ),
251 Document.text_content.ilike(pattern, escape="\\"),
252 ),
253 )
254 .order_by(Document.updated_at.desc())
255 .limit(limit)
256 .all()
257 )
259 results = []
260 for (
261 doc_id,
262 title,
263 filename,
264 preview,
265 source_type_name,
266 updated_at,
267 research_id,
268 pos,
269 ) in rows:
270 preview = preview or ""
271 # Mark the cut ONLY when the window truly opens mid-document, i.e.
272 # preview_start = pos - PREVIEW_CONTEXT > 1, which is pos >
273 # PREVIEW_CONTEXT + 1. At exactly pos == PREVIEW_CONTEXT + 1 the SQL
274 # clamps preview_start to 1 (the document's first char), so the
275 # preview isn't truncated and a leading "…" would be spurious.
276 if pos and pos > PREVIEW_CONTEXT + 1:
277 preview = "…" + preview
278 # Fall back to filename (set by user uploads) when title is NULL,
279 # so uploaded documents without an explicit title still surface a
280 # useful name instead of the generic 'Untitled'. Filename is
281 # already werkzeug-sanitized (spaces -> underscores) at upload
282 # time, so what we return matches what the semantic leg already
283 # shows. Final 'Untitled' fallback only fires when both title
284 # AND filename are missing (defensive — every current code path
285 # sets at least one of them).
286 display_title = title or filename or "Untitled"
287 results.append(
288 {
289 "id": doc_id,
290 "title": display_title,
291 "content_preview": preview,
292 "source_type": source_type_name,
293 "updated_at": updated_at.isoformat()
294 if updated_at
295 else None,
296 "research_id": research_id,
297 "url": _document_url(source_type_name, doc_id, research_id),
298 }
299 )
301 return {
302 "success": True,
303 "query": q,
304 "results": results,
305 "count": len(results),
306 }
308 except Exception as e:
309 return handle_api_error("unified keyword search", e)
312@router.get("/api/semantic")
313@_unified_search_limit
314def semantic_search(
315 request: Request, username: Annotated[str, Depends(require_auth)]
316):
317 """Semantic leg: FAISS search over the system collections.
319 Query params:
320 q: Search query (required, capped at MAX_SEARCH_LEN)
321 limit: Maximum results (default 20, clamped 1..50)
322 min_similarity: Minimum similarity 0.0-1.0 (default 0.25)
324 Searches the RAG index of every existing system collection
325 (default_library, research_history, notes — missing ones are
326 skipped, not created) and merges the hits across collections by
327 similarity desc, one result per document (its best-scoring chunk).
329 Infrastructure failures PROPAGATE as a 500 via ``handle_api_error``
330 — an outage must not masquerade as "no matches" (repo rule: no
331 silent fallbacks). The page's hybrid mode degrades per leg
332 client-side, where the degradation is surfaced as a notice.
333 """
334 try:
335 q, limit, error = _validated_query_and_limit(request)
336 if error:
337 return error
338 try:
339 min_similarity_raw = float(
340 request.query_params.get(
341 "min_similarity", DEFAULT_MIN_SIMILARITY
342 )
343 )
344 # Reject nan/inf explicitly: float("nan") does not raise, and
345 # max(0.0, min(nan, 1.0)) silently collapses to 0.0 (the most
346 # permissive threshold) instead of surfacing the bad input.
347 if not math.isfinite(min_similarity_raw):
348 raise ValueError( # noqa: TRY301 — caught just below and mapped to a 400
349 "min_similarity must be a finite number"
350 )
351 min_similarity = max(0.0, min(min_similarity_raw, 1.0))
352 except (ValueError, TypeError):
353 return JSONResponse(
354 {"success": False, "error": "Invalid min_similarity"},
355 status_code=400,
356 )
358 from ...database.models.library import (
359 Collection,
360 Document,
361 RAGIndex,
362 SourceType,
363 )
364 from ...database.session_context import get_user_db_session
366 # Resolve the searchable collections: the system collections that
367 # exist, plus ANY other collection with a current RAG index. The
368 # keyword leg covers every document, so scoping the semantic leg
369 # to system collections only made the two modes cover different
370 # corpora — documents living solely in an indexed user collection
371 # were keyword-searchable but never AI-searchable. Unindexed
372 # collections are harmless to include (the engine returns [] when
373 # no current index exists), but filtering here avoids paying its
374 # per-collection setup for collections that can't match.
375 with get_user_db_session(username) as db:
376 all_collections = db.query(
377 Collection.id,
378 Collection.name,
379 Collection.collection_type,
380 ).all()
381 indexed_names = {
382 name
383 for (name,) in db.query(RAGIndex.collection_name)
384 .filter(RAGIndex.is_current.is_(True))
385 .all()
386 }
387 collections = [
388 (collection_id, collection_name)
389 for collection_id, collection_name, collection_type in all_collections
390 if collection_type in SEMANTIC_COLLECTION_TYPES
391 or f"collection_{collection_id}" in indexed_names
392 ]
394 if not collections:
395 return {
396 "success": True,
397 "query": q,
398 "results": [],
399 "count": 0,
400 }
402 from ...web_search_engines.engines.search_engine_collection import (
403 CollectionSearchEngine,
404 )
406 # Over-fetch chunk-level hits per collection: documents are
407 # chunk-indexed, so one long document can occupy several top-k
408 # slots before the per-document dedup below.
409 fetch_k = limit * 3
410 hits = []
411 for collection_id, collection_name in collections:
412 engine = CollectionSearchEngine(
413 collection_id=collection_id,
414 collection_name=collection_name,
415 max_results=fetch_k,
416 settings_snapshot={"_username": username},
417 )
418 # Engine failures propagate to handle_api_error below.
419 raw_results = engine.search(q, limit=fetch_k)
420 for r in raw_results or []:
421 score = r.get("relevance_score", 0)
422 if score < min_similarity:
423 continue
424 meta = r.get("metadata", {}) or {}
425 doc_id = meta.get("document_id") or meta.get("source_id")
426 if not doc_id: 426 ↛ 427line 426 didn't jump to line 427 because the condition on line 426 was never true
427 continue
428 hits.append(
429 {
430 "id": doc_id,
431 "title": r.get("title", "Untitled"),
432 "content_preview": (r.get("snippet", "") or "")[
433 :PREVIEW_LEN
434 ],
435 "similarity": float(score),
436 }
437 )
439 # Merge across collections by similarity desc; keep each
440 # document's best-scoring chunk only. Dedup the WHOLE pool (not
441 # just the first `limit`) so the existence check below can backfill
442 # from lower-ranked valid hits when a top hit turns out to be a
443 # FAISS orphan — otherwise a handful of orphans in the top `limit`
444 # silently shrink the result set even though enough valid matches
445 # were fetched.
446 hits.sort(key=lambda h: h["similarity"], reverse=True)
447 candidates = []
448 seen_doc_ids = set()
449 for hit in hits:
450 if hit["id"] in seen_doc_ids:
451 continue
452 seen_doc_ids.add(hit["id"])
453 candidates.append(hit)
455 # Enrich with the Document row (source type + research_id → url).
456 # A candidate absent from the lookup is a FAISS orphan — the
457 # document was deleted but its vectors linger until reindex — so
458 # drop it rather than render a link that 404s, and only cut to
459 # `limit` AFTER dropping orphans.
460 results = []
461 if candidates:
462 with get_user_db_session(username) as db:
463 rows = (
464 db.query(Document.id, SourceType.name, Document.research_id)
465 .join(SourceType, Document.source_type_id == SourceType.id)
466 .filter(
467 Document.id.in_([h["id"] for h in candidates]),
468 # Match the keyword leg: don't surface in-progress
469 # or failed documents even if their vectors exist.
470 Document.status == "completed",
471 )
472 .all()
473 )
474 doc_info = {
475 doc_id: (source_type_name, research_id)
476 for doc_id, source_type_name, research_id in rows
477 }
478 for hit in candidates:
479 info = doc_info.get(hit["id"])
480 if info is None:
481 continue
482 source_type_name, research_id = info
483 hit["source_type"] = source_type_name
484 hit["url"] = _document_url(
485 source_type_name, hit["id"], research_id
486 )
487 hit["similarity"] = round(hit["similarity"], 3)
488 results.append(hit)
489 if len(results) >= limit:
490 break
492 return {
493 "success": True,
494 "query": q,
495 "results": results,
496 "count": len(results),
497 }
499 except Exception as e:
500 return handle_api_error("unified semantic search", e)