Coverage for src/local_deep_research/web/routers/unified_search.py: 98%
130 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""
2Unified Search Routes — "Search everything" (FastAPI).
4One search surface across every entity stored as a ``Document`` row
5(notes, research reports, library downloads, user uploads):
7* GET /library/search/ — the Search page
8* GET /library/search/api/keyword — ILIKE over title + text_content
9* GET /library/search/api/semantic — FAISS search over the system
10 collections (default_library, research_history, notes)
12The two API legs return the SAME result-row contract so the page JS can
13merge them mode-agnostically::
15 {id, title, content_preview, source_type, url, similarity?}
17``url`` is computed server-side (see ``_document_url``) so every future
18consumer links results consistently.
20Ported from the Flask ``web/routes/unified_search_routes.py`` blueprint.
21"""
23import math
25from fastapi import APIRouter, Depends, Request
26from fastapi.responses import JSONResponse
28from ...research_library.utils import escape_like, handle_api_error
29from ..dependencies.auth import require_auth
30from ..dependencies.rate_limit import limiter, _user_key
31from ..template_config import templates
33# MAX_SEARCH_LEN (the user-query cap into ILIKE + the embedding call) is
34# shared with notes_routes via _search_constants so the two route modules
35# can't diverge.
36from ..routes._search_constants import MAX_SEARCH_LEN
37from typing import Annotated
39# Keyword-leg preview length: column-projected ``substr`` so the full
40# text_content (which can be megabytes for extracted PDFs) is never
41# hydrated into the response.
42PREVIEW_LEN = 300
43# Lead-in shown before the first content match when the preview window is
44# anchored on it (see keyword_search).
45PREVIEW_CONTEXT = 60
46DEFAULT_LIMIT = 20
47MAX_LIMIT = 50
48DEFAULT_MIN_SIMILARITY = 0.25
50# Minimum query length, matching the page JS's UNIFIED_SEARCH_MIN_QUERY:
51# below 2 chars the keyword leg is an expensive full-content scan for a
52# near-meaningless match set, and the client never sends one anyway.
53MIN_SEARCH_LEN = 2
55# System collections always searched by the semantic leg, keyed by
56# ``Collection.collection_type``. These are lazily created elsewhere
57# (library init / first note), so a missing one is SKIPPED rather than
58# created — a read endpoint must not have write side effects. User
59# collections join them whenever they have a current RAG index (see
60# semantic_search) so the two legs cover the same corpus for anything
61# the user actually indexed.
62SEMANTIC_COLLECTION_TYPES = ("default_library", "research_history", "notes")
64# Per-user search budget — same shape as the notes_search bucket (cheap
65# DB/FAISS lookups typed through a debounced search box).
66_unified_search_limit = limiter.shared_limit(
67 "60 per minute", scope="unified_search", key_func=_user_key
68)
70router = APIRouter(prefix="/library/search", tags=["unified-search"])
73def _document_url(source_type_name, document_id, research_id):
74 """Canonical link for a search hit, by source type.
76 * ``note`` → the note editor.
77 * ``research_report`` with a ``research_id`` → the research results
78 page (the report Document is a search-index copy of that run).
79 * everything else (uploads, downloads, report copies whose run is
80 gone) → the library document details page.
81 """
82 if source_type_name == "note":
83 return f"/notes/{document_id}"
84 if source_type_name == "research_report" and research_id:
85 return f"/results/{research_id}"
86 return f"/library/document/{document_id}"
89def _validated_query_and_limit(request: Request):
90 """Parse/validate the shared ``q`` and ``limit`` query params.
92 Returns ``(query, limit, error_response)`` where ``error_response``
93 is a ready ``JSONResponse`` when validation failed (and the other
94 fields are None).
95 """
96 q = request.query_params.get("q", "")
97 if len(q) > MAX_SEARCH_LEN:
98 return (
99 None,
100 None,
101 JSONResponse(
102 {
103 "success": False,
104 "error": f"q exceeds maximum length ({MAX_SEARCH_LEN} chars)",
105 },
106 status_code=400,
107 ),
108 )
109 q = q.strip()
110 if not q:
111 return (
112 None,
113 None,
114 JSONResponse(
115 {"success": False, "error": "Query is required"},
116 status_code=400,
117 ),
118 )
119 if len(q) < MIN_SEARCH_LEN:
120 return (
121 None,
122 None,
123 JSONResponse(
124 {
125 "success": False,
126 "error": (
127 f"Query must be at least {MIN_SEARCH_LEN} characters"
128 ),
129 },
130 status_code=400,
131 ),
132 )
133 try:
134 limit = max(
135 1,
136 min(
137 int(request.query_params.get("limit", DEFAULT_LIMIT)), MAX_LIMIT
138 ),
139 )
140 except (ValueError, TypeError):
141 return (
142 None,
143 None,
144 JSONResponse(
145 {"success": False, "error": "Invalid limit"}, status_code=400
146 ),
147 )
148 return q, limit, None
151# ============================================================================
152# Page Route
153# ============================================================================
156@router.get("/")
157def unified_search_page(
158 request: Request, username: Annotated[str, Depends(require_auth)]
159):
160 """Render the unified Search page."""
161 return templates.TemplateResponse(
162 request=request,
163 name="pages/unified_search.html",
164 context={"active_page": "unified-search"},
165 )
168# ============================================================================
169# API Routes
170# ============================================================================
173@router.get("/api/keyword")
174@_unified_search_limit
175def keyword_search(
176 request: Request, username: Annotated[str, Depends(require_auth)]
177):
178 """Keyword leg: ILIKE over Document.title + text_content, ALL source types.
180 Query params:
181 q: Search query (required, capped at MAX_SEARCH_LEN)
182 limit: Maximum results (default 20, clamped 1..50)
184 Column-projected (id, title, filename, 300-char preview, source-type name,
185 updated_at, research_id) — never hydrates full text_content.
186 """
187 try:
188 q, limit, error = _validated_query_and_limit(request)
189 if error:
190 return error
192 from sqlalchemy import case, func, or_
194 from ...database.models.library import Document, SourceType
195 from ...database.session_context import get_user_db_session
197 pattern = f"%{escape_like(q)}%"
198 # Center the preview on the first content match instead of always
199 # slicing the head: for a content hit deep in a long document, the
200 # first 300 chars usually don't contain the matched text at all,
201 # so the card gave no clue why the document matched. instr() takes
202 # the raw query (wildcard escaping only applies to LIKE), folded
203 # with lower() on both sides exactly like ilike; 0 = title-only
204 # match → head preview. PREVIEW_CONTEXT chars of lead-in keep the
205 # match readable in situ.
206 match_pos = func.instr(func.lower(Document.text_content), func.lower(q))
207 preview_start = case(
208 (match_pos > PREVIEW_CONTEXT, match_pos - PREVIEW_CONTEXT),
209 else_=1,
210 )
211 with get_user_db_session(username) as db:
212 rows = (
213 db.query(
214 Document.id,
215 Document.title,
216 Document.filename,
217 func.substr(
218 Document.text_content, preview_start, PREVIEW_LEN
219 ),
220 SourceType.name,
221 Document.updated_at,
222 Document.research_id,
223 match_pos,
224 )
225 .join(SourceType, Document.source_type_id == SourceType.id)
226 .filter(
227 # Only surface completed documents, matching every
228 # library listing/search path (library_service filters
229 # status == 'completed' too). Without this, in-progress
230 # or failed downloads — partial/empty text, or a stale
231 # title — appear as hits that link to a document page
232 # with no usable content.
233 Document.status == "completed",
234 or_(
235 Document.title.ilike(pattern, escape="\\"),
236 Document.text_content.ilike(pattern, escape="\\"),
237 ),
238 )
239 .order_by(Document.updated_at.desc())
240 .limit(limit)
241 .all()
242 )
244 results = []
245 for (
246 doc_id,
247 title,
248 filename,
249 preview,
250 source_type_name,
251 updated_at,
252 research_id,
253 pos,
254 ) in rows:
255 preview = preview or ""
256 # Mark the cut ONLY when the window truly opens mid-document, i.e.
257 # preview_start = pos - PREVIEW_CONTEXT > 1, which is pos >
258 # PREVIEW_CONTEXT + 1. At exactly pos == PREVIEW_CONTEXT + 1 the SQL
259 # clamps preview_start to 1 (the document's first char), so the
260 # preview isn't truncated and a leading "…" would be spurious.
261 if pos and pos > PREVIEW_CONTEXT + 1:
262 preview = "…" + preview
263 # Fall back to filename (set by user uploads) when title is NULL,
264 # so uploaded documents without an explicit title still surface a
265 # useful name instead of the generic 'Untitled'. Filename is
266 # already werkzeug-sanitized (spaces -> underscores) at upload
267 # time, so what we return matches what the semantic leg already
268 # shows. Final 'Untitled' fallback only fires when both title
269 # AND filename are missing (defensive — every current code path
270 # sets at least one of them).
271 display_title = title or filename or "Untitled"
272 results.append(
273 {
274 "id": doc_id,
275 "title": display_title,
276 "content_preview": preview,
277 "source_type": source_type_name,
278 "updated_at": updated_at.isoformat()
279 if updated_at
280 else None,
281 "research_id": research_id,
282 "url": _document_url(source_type_name, doc_id, research_id),
283 }
284 )
286 return {
287 "success": True,
288 "query": q,
289 "results": results,
290 "count": len(results),
291 }
293 except Exception as e:
294 return handle_api_error("unified keyword search", e)
297@router.get("/api/semantic")
298@_unified_search_limit
299def semantic_search(
300 request: Request, username: Annotated[str, Depends(require_auth)]
301):
302 """Semantic leg: FAISS search over the system collections.
304 Query params:
305 q: Search query (required, capped at MAX_SEARCH_LEN)
306 limit: Maximum results (default 20, clamped 1..50)
307 min_similarity: Minimum similarity 0.0-1.0 (default 0.25)
309 Searches the RAG index of every existing system collection
310 (default_library, research_history, notes — missing ones are
311 skipped, not created) and merges the hits across collections by
312 similarity desc, one result per document (its best-scoring chunk).
314 Infrastructure failures PROPAGATE as a 500 via ``handle_api_error``
315 — an outage must not masquerade as "no matches" (repo rule: no
316 silent fallbacks). The page's hybrid mode degrades per leg
317 client-side, where the degradation is surfaced as a notice.
318 """
319 try:
320 q, limit, error = _validated_query_and_limit(request)
321 if error:
322 return error
323 try:
324 min_similarity_raw = float(
325 request.query_params.get(
326 "min_similarity", DEFAULT_MIN_SIMILARITY
327 )
328 )
329 # Reject nan/inf explicitly: float("nan") does not raise, and
330 # max(0.0, min(nan, 1.0)) silently collapses to 0.0 (the most
331 # permissive threshold) instead of surfacing the bad input.
332 if not math.isfinite(min_similarity_raw):
333 raise ValueError( # noqa: TRY301 — caught just below and mapped to a 400
334 "min_similarity must be a finite number"
335 )
336 min_similarity = max(0.0, min(min_similarity_raw, 1.0))
337 except (ValueError, TypeError):
338 return JSONResponse(
339 {"success": False, "error": "Invalid min_similarity"},
340 status_code=400,
341 )
343 from ...database.models.library import (
344 Collection,
345 Document,
346 RAGIndex,
347 SourceType,
348 )
349 from ...database.session_context import get_user_db_session
351 # Resolve the searchable collections: the system collections that
352 # exist, plus ANY other collection with a current RAG index. The
353 # keyword leg covers every document, so scoping the semantic leg
354 # to system collections only made the two modes cover different
355 # corpora — documents living solely in an indexed user collection
356 # were keyword-searchable but never AI-searchable. Unindexed
357 # collections are harmless to include (the engine returns [] when
358 # no current index exists), but filtering here avoids paying its
359 # per-collection setup for collections that can't match.
360 with get_user_db_session(username) as db:
361 all_collections = db.query(
362 Collection.id,
363 Collection.name,
364 Collection.collection_type,
365 ).all()
366 indexed_names = {
367 name
368 for (name,) in db.query(RAGIndex.collection_name)
369 .filter(RAGIndex.is_current.is_(True))
370 .all()
371 }
372 collections = [
373 (collection_id, collection_name)
374 for collection_id, collection_name, collection_type in all_collections
375 if collection_type in SEMANTIC_COLLECTION_TYPES
376 or f"collection_{collection_id}" in indexed_names
377 ]
379 if not collections:
380 return {
381 "success": True,
382 "query": q,
383 "results": [],
384 "count": 0,
385 }
387 from ...web_search_engines.engines.search_engine_collection import (
388 CollectionSearchEngine,
389 )
391 # Over-fetch chunk-level hits per collection: documents are
392 # chunk-indexed, so one long document can occupy several top-k
393 # slots before the per-document dedup below.
394 fetch_k = limit * 3
395 hits = []
396 for collection_id, collection_name in collections:
397 engine = CollectionSearchEngine(
398 collection_id=collection_id,
399 collection_name=collection_name,
400 max_results=fetch_k,
401 settings_snapshot={"_username": username},
402 )
403 # Engine failures propagate to handle_api_error below.
404 raw_results = engine.search(q, limit=fetch_k)
405 for r in raw_results or []:
406 score = r.get("relevance_score", 0)
407 if score < min_similarity:
408 continue
409 meta = r.get("metadata", {}) or {}
410 doc_id = meta.get("document_id") or meta.get("source_id")
411 if not doc_id: 411 ↛ 412line 411 didn't jump to line 412 because the condition on line 411 was never true
412 continue
413 hits.append(
414 {
415 "id": doc_id,
416 "title": r.get("title", "Untitled"),
417 "content_preview": (r.get("snippet", "") or "")[
418 :PREVIEW_LEN
419 ],
420 "similarity": float(score),
421 }
422 )
424 # Merge across collections by similarity desc; keep each
425 # document's best-scoring chunk only. Dedup the WHOLE pool (not
426 # just the first `limit`) so the existence check below can backfill
427 # from lower-ranked valid hits when a top hit turns out to be a
428 # FAISS orphan — otherwise a handful of orphans in the top `limit`
429 # silently shrink the result set even though enough valid matches
430 # were fetched.
431 hits.sort(key=lambda h: h["similarity"], reverse=True)
432 candidates = []
433 seen_doc_ids = set()
434 for hit in hits:
435 if hit["id"] in seen_doc_ids:
436 continue
437 seen_doc_ids.add(hit["id"])
438 candidates.append(hit)
440 # Enrich with the Document row (source type + research_id → url).
441 # A candidate absent from the lookup is a FAISS orphan — the
442 # document was deleted but its vectors linger until reindex — so
443 # drop it rather than render a link that 404s, and only cut to
444 # `limit` AFTER dropping orphans.
445 results = []
446 if candidates:
447 with get_user_db_session(username) as db:
448 rows = (
449 db.query(Document.id, SourceType.name, Document.research_id)
450 .join(SourceType, Document.source_type_id == SourceType.id)
451 .filter(
452 Document.id.in_([h["id"] for h in candidates]),
453 # Match the keyword leg: don't surface in-progress
454 # or failed documents even if their vectors exist.
455 Document.status == "completed",
456 )
457 .all()
458 )
459 doc_info = {
460 doc_id: (source_type_name, research_id)
461 for doc_id, source_type_name, research_id in rows
462 }
463 for hit in candidates:
464 info = doc_info.get(hit["id"])
465 if info is None:
466 continue
467 source_type_name, research_id = info
468 hit["source_type"] = source_type_name
469 hit["url"] = _document_url(
470 source_type_name, hit["id"], research_id
471 )
472 hit["similarity"] = round(hit["similarity"], 3)
473 results.append(hit)
474 if len(results) >= limit:
475 break
477 return {
478 "success": True,
479 "query": q,
480 "results": results,
481 "count": len(results),
482 }
484 except Exception as e:
485 return handle_api_error("unified semantic search", e)