Coverage for src/local_deep_research/web/routes/unified_search_routes.py: 95%
135 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-19 23:35 +0000
1"""
2Unified Search Routes — "Search everything".
4One search surface across every entity stored as a ``Document`` row
5(notes, research reports, library downloads, user uploads):
7* GET /library/search/ — the Search page
8* GET /library/search/api/keyword — ILIKE over title + text_content
9* GET /library/search/api/semantic — FAISS search over the system
10 collections (default_library, research_history, notes)
12The two API legs return the SAME result-row contract so the page JS can
13merge them mode-agnostically::
15 {id, title, content_preview, source_type, url, similarity?}
17``url`` is computed server-side (see ``_document_url``) so every future
18consumer links results consistently.
19"""
21import math
23from flask import Blueprint, jsonify, request, session
25from ...research_library.utils import escape_like, handle_api_error
26from ...security.rate_limiter import limiter, _get_api_user_key
27from ..auth.decorators import login_required
28from ..utils.templates import render_template_with_defaults
30# MAX_SEARCH_LEN (the user-query cap into ILIKE + the embedding call) is
31# shared with notes_routes via _search_constants so the two route modules
32# can't diverge.
33from ._search_constants import MAX_SEARCH_LEN
35# Keyword-leg preview length: column-projected ``substr`` so the full
36# text_content (which can be megabytes for extracted PDFs) is never
37# hydrated into the response.
38PREVIEW_LEN = 300
39# Lead-in shown before the first content match when the preview window is
40# anchored on it (see keyword_search).
41PREVIEW_CONTEXT = 60
42DEFAULT_LIMIT = 20
43MAX_LIMIT = 50
44DEFAULT_MIN_SIMILARITY = 0.25
46# Minimum query length, matching the page JS's UNIFIED_SEARCH_MIN_QUERY:
47# below 2 chars the keyword leg is an expensive full-content scan for a
48# near-meaningless match set, and the client never sends one anyway.
49MIN_SEARCH_LEN = 2
51# System collections always searched by the semantic leg, keyed by
52# ``Collection.collection_type``. These are lazily created elsewhere
53# (library init / first note), so a missing one is SKIPPED rather than
54# created — a read endpoint must not have write side effects. User
55# collections join them whenever they have a current RAG index (see
56# semantic_search) so the two legs cover the same corpus for anything
57# the user actually indexed.
58SEMANTIC_COLLECTION_TYPES = ("default_library", "research_history", "notes")
60# Per-user search budget — same shape as the notes_search bucket (cheap
61# DB/FAISS lookups typed through a debounced search box).
62_unified_search_limit = limiter.shared_limit(
63 "60 per minute", scope="unified_search", key_func=_get_api_user_key
64)
66unified_search_bp = Blueprint(
67 "unified_search", __name__, url_prefix="/library/search"
68)
71def _document_url(source_type_name, document_id, research_id):
72 """Canonical link for a search hit, by source type.
74 * ``note`` → the note editor.
75 * ``research_report`` with a ``research_id`` → the research results
76 page (the report Document is a search-index copy of that run).
77 * everything else (uploads, downloads, report copies whose run is
78 gone) → the library document details page.
79 """
80 if source_type_name == "note":
81 return f"/notes/{document_id}"
82 if source_type_name == "research_report" and research_id:
83 return f"/results/{research_id}"
84 return f"/library/document/{document_id}"
87def _validated_query_and_limit():
88 """Parse/validate the shared ``q`` and ``limit`` query params.
90 Returns ``(query, limit, error_response)`` where ``error_response``
91 is a ready ``(json, status)`` tuple when validation failed (and the
92 other fields are None).
93 """
94 q = request.args.get("q", "")
95 if len(q) > MAX_SEARCH_LEN:
96 return (
97 None,
98 None,
99 (
100 jsonify(
101 {
102 "success": False,
103 "error": f"q exceeds maximum length ({MAX_SEARCH_LEN} chars)",
104 }
105 ),
106 400,
107 ),
108 )
109 q = q.strip()
110 if not q:
111 return (
112 None,
113 None,
114 (jsonify({"success": False, "error": "Query is required"}), 400),
115 )
116 if len(q) < MIN_SEARCH_LEN:
117 return (
118 None,
119 None,
120 (
121 jsonify(
122 {
123 "success": False,
124 "error": (
125 f"Query must be at least {MIN_SEARCH_LEN} "
126 "characters"
127 ),
128 }
129 ),
130 400,
131 ),
132 )
133 try:
134 limit = max(
135 1, min(int(request.args.get("limit", DEFAULT_LIMIT)), MAX_LIMIT)
136 )
137 except (ValueError, TypeError):
138 return (
139 None,
140 None,
141 (jsonify({"success": False, "error": "Invalid limit"}), 400),
142 )
143 return q, limit, None
146# ============================================================================
147# Page Route
148# ============================================================================
151@unified_search_bp.route("/")
152@login_required
153def unified_search_page():
154 """Render the unified Search page."""
155 return render_template_with_defaults(
156 "pages/unified_search.html", active_page="unified-search"
157 )
160# ============================================================================
161# API Routes
162# ============================================================================
165@unified_search_bp.route("/api/keyword", methods=["GET"])
166@login_required
167@_unified_search_limit
168def keyword_search():
169 """Keyword leg: ILIKE over Document.title + text_content, ALL source types.
171 Query params:
172 q: Search query (required, capped at MAX_SEARCH_LEN)
173 limit: Maximum results (default 20, clamped 1..50)
175 Column-projected (id, title, 300-char preview, source-type name,
176 updated_at, research_id) — never hydrates full text_content.
177 """
178 username = session.get("username")
179 if not username: 179 ↛ 180line 179 didn't jump to line 180 because the condition on line 179 was never true
180 return jsonify({"success": False, "error": "Not authenticated"}), 401
182 try:
183 q, limit, error = _validated_query_and_limit()
184 if error:
185 return error
187 from sqlalchemy import case, func, or_
189 from ...database.models.library import Document, SourceType
190 from ...database.session_context import get_user_db_session
192 pattern = f"%{escape_like(q)}%"
193 # Center the preview on the first content match instead of always
194 # slicing the head: for a content hit deep in a long document, the
195 # first 300 chars usually don't contain the matched text at all,
196 # so the card gave no clue why the document matched. instr() takes
197 # the raw query (wildcard escaping only applies to LIKE), folded
198 # with lower() on both sides exactly like ilike; 0 = title-only
199 # match → head preview. PREVIEW_CONTEXT chars of lead-in keep the
200 # match readable in situ.
201 match_pos = func.instr(func.lower(Document.text_content), func.lower(q))
202 preview_start = case(
203 (match_pos > PREVIEW_CONTEXT, match_pos - PREVIEW_CONTEXT),
204 else_=1,
205 )
206 with get_user_db_session(username) as db:
207 rows = (
208 db.query(
209 Document.id,
210 Document.title,
211 func.substr(
212 Document.text_content, preview_start, PREVIEW_LEN
213 ),
214 SourceType.name,
215 Document.updated_at,
216 Document.research_id,
217 match_pos,
218 )
219 .join(SourceType, Document.source_type_id == SourceType.id)
220 .filter(
221 # Only surface completed documents, matching every
222 # library listing/search path (library_service filters
223 # status == 'completed' too). Without this, in-progress
224 # or failed downloads — partial/empty text, or a stale
225 # title — appear as hits that link to a document page
226 # with no usable content.
227 Document.status == "completed",
228 or_(
229 Document.title.ilike(pattern, escape="\\"),
230 Document.text_content.ilike(pattern, escape="\\"),
231 ),
232 )
233 .order_by(Document.updated_at.desc())
234 .limit(limit)
235 .all()
236 )
238 results = []
239 for (
240 doc_id,
241 title,
242 preview,
243 source_type_name,
244 updated_at,
245 research_id,
246 pos,
247 ) in rows:
248 preview = preview or ""
249 # Mark the cut ONLY when the window truly opens mid-document, i.e.
250 # preview_start = pos - PREVIEW_CONTEXT > 1, which is pos >
251 # PREVIEW_CONTEXT + 1. At exactly pos == PREVIEW_CONTEXT + 1 the SQL
252 # clamps preview_start to 1 (the document's first char), so the
253 # preview isn't truncated and a leading "…" would be spurious.
254 if pos and pos > PREVIEW_CONTEXT + 1:
255 preview = "…" + preview
256 results.append(
257 {
258 "id": doc_id,
259 "title": title or "Untitled",
260 "content_preview": preview,
261 "source_type": source_type_name,
262 "updated_at": updated_at.isoformat()
263 if updated_at
264 else None,
265 "research_id": research_id,
266 "url": _document_url(source_type_name, doc_id, research_id),
267 }
268 )
270 return jsonify(
271 {
272 "success": True,
273 "query": q,
274 "results": results,
275 "count": len(results),
276 }
277 )
279 except Exception as e:
280 return handle_api_error("unified keyword search", e)
283@unified_search_bp.route("/api/semantic", methods=["GET"])
284@login_required
285@_unified_search_limit
286def semantic_search():
287 """Semantic leg: FAISS search over the system collections.
289 Query params:
290 q: Search query (required, capped at MAX_SEARCH_LEN)
291 limit: Maximum results (default 20, clamped 1..50)
292 min_similarity: Minimum similarity 0.0-1.0 (default 0.25)
294 Searches the RAG index of every existing system collection
295 (default_library, research_history, notes — missing ones are
296 skipped, not created) and merges the hits across collections by
297 similarity desc, one result per document (its best-scoring chunk).
299 Infrastructure failures PROPAGATE as a 500 via ``handle_api_error``
300 — an outage must not masquerade as "no matches" (repo rule: no
301 silent fallbacks). The page's hybrid mode degrades per leg
302 client-side, where the degradation is surfaced as a notice.
303 """
304 username = session.get("username")
305 if not username: 305 ↛ 306line 305 didn't jump to line 306 because the condition on line 305 was never true
306 return jsonify({"success": False, "error": "Not authenticated"}), 401
308 try:
309 q, limit, error = _validated_query_and_limit()
310 if error:
311 return error
312 try:
313 min_similarity_raw = float(
314 request.args.get("min_similarity", DEFAULT_MIN_SIMILARITY)
315 )
316 # Reject nan/inf explicitly: float("nan") does not raise, and
317 # max(0.0, min(nan, 1.0)) silently collapses to 0.0 (the most
318 # permissive threshold) instead of surfacing the bad input.
319 if not math.isfinite(min_similarity_raw):
320 raise ValueError( # noqa: TRY301 — caught just below and mapped to a 400
321 "min_similarity must be a finite number"
322 )
323 min_similarity = max(0.0, min(min_similarity_raw, 1.0))
324 except (ValueError, TypeError):
325 return jsonify(
326 {"success": False, "error": "Invalid min_similarity"}
327 ), 400
329 from ...database.models.library import (
330 Collection,
331 Document,
332 RAGIndex,
333 SourceType,
334 )
335 from ...database.session_context import get_user_db_session
337 # Resolve the searchable collections: the system collections that
338 # exist, plus ANY other collection with a current RAG index. The
339 # keyword leg covers every document, so scoping the semantic leg
340 # to system collections only made the two modes cover different
341 # corpora — documents living solely in an indexed user collection
342 # were keyword-searchable but never AI-searchable. Unindexed
343 # collections are harmless to include (the engine returns [] when
344 # no current index exists), but filtering here avoids paying its
345 # per-collection setup for collections that can't match.
346 with get_user_db_session(username) as db:
347 all_collections = db.query(
348 Collection.id,
349 Collection.name,
350 Collection.collection_type,
351 ).all()
352 indexed_names = {
353 name
354 for (name,) in db.query(RAGIndex.collection_name)
355 .filter(RAGIndex.is_current.is_(True))
356 .all()
357 }
358 collections = [
359 (collection_id, collection_name)
360 for collection_id, collection_name, collection_type in all_collections
361 if collection_type in SEMANTIC_COLLECTION_TYPES
362 or f"collection_{collection_id}" in indexed_names
363 ]
365 if not collections:
366 return jsonify(
367 {"success": True, "query": q, "results": [], "count": 0}
368 )
370 from ...web_search_engines.engines.search_engine_collection import (
371 CollectionSearchEngine,
372 )
374 # Over-fetch chunk-level hits per collection: documents are
375 # chunk-indexed, so one long document can occupy several top-k
376 # slots before the per-document dedup below.
377 fetch_k = limit * 3
378 hits = []
379 for collection_id, collection_name in collections:
380 engine = CollectionSearchEngine(
381 collection_id=collection_id,
382 collection_name=collection_name,
383 max_results=fetch_k,
384 settings_snapshot={"_username": username},
385 )
386 # Engine failures propagate to handle_api_error below.
387 raw_results = engine.search(q, limit=fetch_k)
388 for r in raw_results or []:
389 score = r.get("relevance_score", 0)
390 if score < min_similarity:
391 continue
392 meta = r.get("metadata", {}) or {}
393 doc_id = meta.get("document_id") or meta.get("source_id")
394 if not doc_id: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 continue
396 hits.append(
397 {
398 "id": doc_id,
399 "title": r.get("title", "Untitled"),
400 "content_preview": (r.get("snippet", "") or "")[
401 :PREVIEW_LEN
402 ],
403 "similarity": round(float(score), 3),
404 }
405 )
407 # Merge across collections by similarity desc; keep each
408 # document's best-scoring chunk only. Dedup the WHOLE pool (not
409 # just the first `limit`) so the existence check below can backfill
410 # from lower-ranked valid hits when a top hit turns out to be a
411 # FAISS orphan — otherwise a handful of orphans in the top `limit`
412 # silently shrink the result set even though enough valid matches
413 # were fetched.
414 hits.sort(key=lambda h: h["similarity"], reverse=True)
415 candidates = []
416 seen_doc_ids = set()
417 for hit in hits:
418 if hit["id"] in seen_doc_ids:
419 continue
420 seen_doc_ids.add(hit["id"])
421 candidates.append(hit)
423 # Enrich with the Document row (source type + research_id → url).
424 # A candidate absent from the lookup is a FAISS orphan — the
425 # document was deleted but its vectors linger until reindex — so
426 # drop it rather than render a link that 404s, and only cut to
427 # `limit` AFTER dropping orphans.
428 results = []
429 if candidates:
430 with get_user_db_session(username) as db:
431 rows = (
432 db.query(Document.id, SourceType.name, Document.research_id)
433 .join(SourceType, Document.source_type_id == SourceType.id)
434 .filter(
435 Document.id.in_([h["id"] for h in candidates]),
436 # Match the keyword leg: don't surface in-progress
437 # or failed documents even if their vectors exist.
438 Document.status == "completed",
439 )
440 .all()
441 )
442 doc_info = {
443 doc_id: (source_type_name, research_id)
444 for doc_id, source_type_name, research_id in rows
445 }
446 for hit in candidates:
447 info = doc_info.get(hit["id"])
448 if info is None:
449 continue
450 source_type_name, research_id = info
451 hit["source_type"] = source_type_name
452 hit["url"] = _document_url(
453 source_type_name, hit["id"], research_id
454 )
455 results.append(hit)
456 if len(results) >= limit:
457 break
459 return jsonify(
460 {
461 "success": True,
462 "query": q,
463 "results": results,
464 "count": len(results),
465 }
466 )
468 except Exception as e:
469 return handle_api_error("unified semantic search", e)