Coverage for src/local_deep_research/web/routers/history.py: 100%
175 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1from fastapi import APIRouter, Depends, Request
2from fastapi.responses import JSONResponse
3from ..dependencies.auth import require_auth
4from ..dependencies.rate_limit import limiter
5from ..template_config import templates
6from ..utils.pagination import OFFSET_LIMIT_ERROR, parse_bounded_offset
8import json
10from loguru import logger
11from .notes import _log_value_preview
12from sqlalchemy import func
14from ...constants import (
15 HISTORY_LOGS_DEFAULT_LIMIT,
16 HISTORY_LOGS_HARD_CAP,
17 ResearchStatus,
18)
19from ...database.models import ResearchHistory
20from ...database.models.library import Document as Document
21from ...database.session_context import get_user_db_session
23from ..models.database import (
24 get_logs_for_research,
25 get_total_logs_for_research,
26)
27from ..research_state import get_active_research_snapshot
28from ..services.research_service import get_research_strategy
29from .research import _research_not_found
31from ...security import filter_research_metadata, strip_settings_snapshot
32from typing import Annotated
34# Create the router for the history routes
35router = APIRouter(prefix="/history", tags=["history"])
37# NOTE: Routes use username (not .get()) intentionally.
38# Depends(require_auth) guarantees the key exists; direct access fails
39# fast if the dependency is ever removed.
41# resolve_report_path removed - reports are now stored in database
44@router.get("/")
45def history_page(
46 request: Request, username: Annotated[str, Depends(require_auth)]
47):
48 """Render the history page"""
49 return templates.TemplateResponse(
50 request=request, name="pages/history.html", context={"request": request}
51 )
54@router.get("/api")
55def get_history(
56 request: Request, username: Annotated[str, Depends(require_auth)]
57):
58 """Get the research history JSON data"""
60 # Flask parsed these with `type=int`, which falls back to the default on
61 # a non-integer instead of raising. Bare int() inside the outer try meant
62 # `?limit=abc` produced a 500 rather than a 200 with defaults. The sibling
63 # endpoint kept the guard — see the clamp in library.py's
64 # get_research_documents, whose comment even points back here — so this
65 # was the odd one out, not a deliberate change.
66 try:
67 limit = max(1, min(int(request.query_params.get("limit", 200)), 500))
68 except (TypeError, ValueError):
69 limit = 200
70 try:
71 offset = parse_bounded_offset(request.query_params.get("offset"))
72 except ValueError:
73 return JSONResponse({"error": OFFSET_LIMIT_ERROR}, status_code=400)
75 try:
76 with get_user_db_session(username) as db_session:
77 # Single query with JOIN to get history + document counts.
78 # Select only the columns the response needs — never the
79 # large ``report_content`` Text body — so listing history
80 # doesn't pull every report into memory (#4560).
81 results = (
82 db_session.query(
83 ResearchHistory.id,
84 ResearchHistory.title,
85 ResearchHistory.query,
86 ResearchHistory.mode,
87 ResearchHistory.status,
88 ResearchHistory.created_at,
89 ResearchHistory.completed_at,
90 ResearchHistory.duration_seconds,
91 ResearchHistory.research_meta,
92 ResearchHistory.chat_session_id,
93 func.count(Document.id).label("document_count"),
94 )
95 .outerjoin(Document, Document.research_id == ResearchHistory.id)
96 .group_by(ResearchHistory.id)
97 .order_by(ResearchHistory.created_at.desc())
98 .limit(limit)
99 .offset(offset)
100 .all()
101 )
103 logger.debug(f"All research count: {len(results)}")
105 # Convert to list of dicts
106 history = []
107 for research in results:
108 item = {
109 "id": research.id,
110 "title": research.title,
111 "query": research.query,
112 "mode": research.mode,
113 "status": research.status,
114 "created_at": research.created_at,
115 "completed_at": research.completed_at,
116 "duration_seconds": research.duration_seconds,
117 "document_count": research.document_count,
118 }
120 item["metadata"] = filter_research_metadata(
121 research.research_meta
122 )
123 if research.chat_session_id is not None:
124 item["metadata"]["chat_session_id"] = (
125 research.chat_session_id
126 )
128 # Recalculate duration if null but both timestamps exist
129 if (
130 item["duration_seconds"] is None
131 and item["created_at"]
132 and item["completed_at"]
133 ):
134 try:
135 from dateutil import parser # type: ignore[import-untyped]
137 start_time = parser.parse(item["created_at"])
138 end_time = parser.parse(item["completed_at"])
139 item["duration_seconds"] = int(
140 (end_time - start_time).total_seconds()
141 )
142 except Exception:
143 logger.warning("Error recalculating duration")
144 logger.debug("Duration error details", exc_info=True)
146 history.append(item)
148 # Format response to match what client expects
149 return {
150 "status": "success",
151 "items": history, # Use 'items' key as expected by client
152 }
154 # CORS headers are handled by SecurityHeaders middleware
155 except Exception:
156 logger.exception("Error getting history")
157 # 500, not a bare dict: a bare return serializes as 200, so the
158 # client's `response.ok` check passes and it renders an empty
159 # history as if the user had none. Matches every other error path
160 # in this file.
161 return JSONResponse(
162 {
163 "status": "error",
164 "items": [],
165 "message": "Failed to retrieve history",
166 },
167 status_code=500,
168 )
171def _scrub_server_paths(value):
172 """Recursively drop server-path keys from a parsed progress-log entry.
174 Rows written by <=v0.4.4 persisted the completion entry with
175 ``metadata.report_path`` — the absolute server-side path of the
176 generated report — inside the ``progress_log`` column. The raw
177 column itself is never returned to the client; this scrub is
178 defence in depth for the parsed entries returned as ``log``, in
179 case a path-bearing key is ever nested back into the column.
180 ``report_path`` is the only such key any writer has ever emitted.
181 """
182 if isinstance(value, dict):
183 return {
184 k: _scrub_server_paths(v)
185 for k, v in value.items()
186 if k != "report_path"
187 }
188 if isinstance(value, list):
189 return [_scrub_server_paths(item) for item in value]
190 return value
193@router.get("/status/{research_id}")
194@limiter.exempt
195def get_research_status(
196 request: Request,
197 research_id,
198 username: Annotated[str, Depends(require_auth)],
199):
201 with get_user_db_session(username) as db_session:
202 research = (
203 db_session.query(ResearchHistory).filter_by(id=research_id).first()
204 )
206 if not research:
207 return _research_not_found(research_id)
209 # Extract attributes while session is active
210 # to avoid DetachedInstanceError after the with block exits
211 raw_progress_log = research.progress_log
212 result = {
213 "id": research.id,
214 "query": research.query,
215 "mode": research.mode,
216 "status": research.status,
217 "created_at": research.created_at,
218 "completed_at": research.completed_at,
219 # report_path deliberately omitted: it's the absolute
220 # server-side path, which leaks directory layout — same rule
221 # as /library/api/check-downloads and the history LIST
222 # endpoint. progress_log is omitted for the same reason:
223 # rows written by <=v0.4.4 embed report_path inside the
224 # completion entry's metadata, and no client reads the raw
225 # field — the sanitized parsed form is returned as "log".
226 }
228 # Add progress information from active research (atomic snapshot)
229 snapshot = get_active_research_snapshot(research_id)
230 if snapshot is not None:
231 result["progress"] = snapshot["progress"]
232 result["log"] = snapshot["log"]
233 elif result.get("status") == ResearchStatus.COMPLETED:
234 result["progress"] = 100
235 try:
236 result["log"] = json.loads(raw_progress_log or "[]")
237 except Exception:
238 logger.warning(
239 "Error parsing progress_log for research {}", research_id
240 )
241 result["log"] = []
242 else:
243 result["progress"] = 0
244 try:
245 result["log"] = json.loads(raw_progress_log or "[]")
246 except Exception:
247 logger.warning(
248 "Error parsing progress_log for research {}", research_id
249 )
250 result["log"] = []
252 # Scrub once at egress so the invariant holds structurally for every
253 # branch above, instead of being enumerated at each assignment site.
254 # This also covers the snapshot branch's "log": its only writer
255 # (the termination handler in research.py) has never nested
256 # report_path into an entry's metadata, so scrubbing it is a no-op
257 # today — and stays defence in depth if that ever changes.
258 result["log"] = _scrub_server_paths(result["log"])
260 return result
263@router.get("/details/{research_id}")
264def get_research_details(
265 request: Request,
266 research_id,
267 username: Annotated[str, Depends(require_auth)],
268):
269 """Get detailed progress log for a specific research"""
271 logger.opt(lazy=True).debug(
272 "Details route accessed for research_id: {}",
273 lambda: _log_value_preview(research_id),
274 )
276 try:
277 with get_user_db_session(username) as db_session:
278 research = (
279 db_session.query(ResearchHistory)
280 .filter_by(id=research_id)
281 .first()
282 )
283 logger.debug(f"Research found: {research.id if research else None}")
285 if not research:
286 logger.error(
287 f"Research not found for id: {_log_value_preview(research_id)}"
288 )
289 return _research_not_found(research_id)
291 # Extract all needed attributes while session is active
292 # to avoid DetachedInstanceError after the with block exits
293 research_data = {
294 "query": research.query,
295 "mode": research.mode,
296 "status": research.status,
297 "created_at": research.created_at,
298 "completed_at": research.completed_at,
299 }
300 except Exception:
301 logger.exception("Database error")
302 return JSONResponse(
303 {
304 "status": "error",
305 "message": "An internal database error occurred.",
306 },
307 status_code=500,
308 )
310 # Get logs from the dedicated log database
311 logs = get_logs_for_research(research_id, username)
313 # Get strategy information
314 strategy_name = get_research_strategy(research_id, username=username)
316 # Get an atomic snapshot of active research state
317 snapshot = get_active_research_snapshot(research_id)
319 # If this is an active research, merge with any in-memory logs
320 if snapshot is not None:
321 # Use the logs from memory temporarily until they're saved to the database
322 memory_logs = snapshot["log"]
324 # Filter out logs that are already in the database by timestamp
325 db_timestamps = {log["time"] for log in logs}
326 unique_memory_logs = [
327 log for log in memory_logs if log["time"] not in db_timestamps
328 ]
330 # Add unique memory logs to our return list
331 logs.extend(unique_memory_logs)
333 # Sort logs by timestamp
334 logs.sort(key=lambda x: x["time"])
336 progress = (
337 snapshot["progress"]
338 if snapshot is not None
339 else (100 if research_data["status"] == ResearchStatus.COMPLETED else 0)
340 )
342 return {
343 "research_id": research_id,
344 "query": research_data["query"],
345 "mode": research_data["mode"],
346 "status": research_data["status"],
347 "strategy": strategy_name,
348 "progress": progress,
349 "created_at": research_data["created_at"],
350 "completed_at": research_data["completed_at"],
351 "log": logs,
352 }
355@router.get("/report/{research_id}")
356def get_report(
357 request: Request,
358 research_id,
359 username: Annotated[str, Depends(require_auth)],
360):
361 with get_user_db_session(username) as db_session:
362 research = (
363 db_session.query(ResearchHistory).filter_by(id=research_id).first()
364 )
366 if not research:
367 return _research_not_found(research_id, message="Report not found")
369 try:
370 # research.report_content holds the answer-only string; rebuild
371 # the legacy display shape (answer + Sources from
372 # research_resources + Metrics from research_meta) on demand.
373 # Do NOT read storage.get_report*() here — those return the
374 # answer-only body (notification-teaser shape) and would drop the
375 # Sources / Research Metrics sections (#3665).
376 from ..services.report_assembly_service import assemble_full_report
378 content = assemble_full_report(research, db_session)
379 # Only None means "research not found" — guarded above. An empty
380 # but found row returns "" and is a valid response.
381 if content is None:
382 return _research_not_found(
383 research_id, message="Report content not found"
384 )
386 # Strip settings_snapshot (API keys, tokens, base URLs, paths)
387 # before returning research_meta to the client — settings_snapshot
388 # is persisted in research_meta but must never reach the API
389 # response. Matches the sibling /details and /api/research/<id>
390 # routes; all other metadata fields are preserved.
391 stored_metadata = strip_settings_snapshot(research.research_meta)
393 # Create an enhanced metadata dictionary with database fields
394 enhanced_metadata = {
395 "query": research.query,
396 "mode": research.mode,
397 "created_at": research.created_at,
398 "completed_at": research.completed_at,
399 "duration": research.duration_seconds,
400 }
402 # Merge with stored metadata
403 enhanced_metadata.update(stored_metadata)
405 return {
406 "status": "success",
407 "content": content,
408 "query": research.query,
409 "mode": research.mode,
410 "created_at": research.created_at,
411 "completed_at": research.completed_at,
412 "metadata": enhanced_metadata,
413 }
415 except Exception:
416 logger.exception(
417 "Failed to retrieve report for research {}", research_id
418 )
419 return JSONResponse(
420 {"status": "error", "message": "Failed to retrieve report"},
421 status_code=500,
422 )
425@router.get("/markdown/{research_id}")
426def get_markdown(
427 request: Request,
428 research_id,
429 username: Annotated[str, Depends(require_auth)],
430):
431 """Get markdown export for a specific research"""
432 with get_user_db_session(username) as db_session:
433 research = (
434 db_session.query(ResearchHistory).filter_by(id=research_id).first()
435 )
437 if not research:
438 return _research_not_found(research_id, message="Report not found")
440 try:
441 # Rebuild the assembled legacy shape (answer + Sources + Metrics)
442 # rather than the answer-only storage.get_report() body (#3665).
443 from ..services.report_assembly_service import assemble_full_report
445 content = assemble_full_report(research, db_session)
446 if content is None:
447 return _research_not_found(
448 research_id, message="Report content not found"
449 )
451 return {"status": "success", "content": content}
452 except Exception:
453 logger.exception(
454 "Failed to retrieve markdown report for research {}",
455 research_id,
456 )
457 return JSONResponse(
458 {"status": "error", "message": "Failed to retrieve report"},
459 status_code=500,
460 )
463@router.get("/logs/{research_id}")
464def get_research_logs(
465 request: Request,
466 research_id,
467 username: Annotated[str, Depends(require_auth)],
468):
469 """Get logs for a specific research ID.
471 Accepts ``?limit=N`` to bound the response size; the default matches
472 the frontend's ``MAX_LOG_ENTRIES`` DOM cap. Clamped to
473 ``[1, HISTORY_LOGS_HARD_CAP]`` so a client cannot force an unbounded
474 load (a long langgraph run can persist thousands of rows; pre-cap
475 the route allocated ~150 MB transient on the server and Firefox
476 parsed a ~50 MB JSON response).
477 """
479 # Per-request cap. HISTORY_LOGS_DEFAULT_LIMIT (500) matches
480 # MAX_LOG_ENTRIES in logpanel.js; the HISTORY_LOGS_HARD_CAP (5000)
481 # ceiling lets explicit log-download flows still get more rows but
482 # stops accidental unbounded loads.
483 # Parse defensively: a non-numeric ?limit= must not 500 (Werkzeug's
484 # request.args.get(type=int) silently fell back to the default; the
485 # bare int() here would raise ValueError -> 500).
486 try:
487 limit = int(
488 request.query_params.get("limit", str(HISTORY_LOGS_DEFAULT_LIMIT))
489 )
490 except (TypeError, ValueError):
491 limit = HISTORY_LOGS_DEFAULT_LIMIT
492 limit = max(1, min(limit, HISTORY_LOGS_HARD_CAP))
494 # First check if the research exists
495 with get_user_db_session(username) as db_session:
496 research = (
497 db_session.query(ResearchHistory).filter_by(id=research_id).first()
498 )
500 if not research:
501 return _research_not_found(research_id)
503 # Retrieve logs from the database
504 logs = get_logs_for_research(research_id, username, limit=limit)
506 # Defensive backfill for any row missing the three frontend-required
507 # fields. `get_logs_for_research` always sets these from ResearchLog
508 # columns, so this backfill is belt-and-braces. It is currently
509 # UNCOVERED: the test that exercised it,
510 # test_logs_with_missing_fields_get_defaults, was removed with the Flask
511 # suite and has no successor (0 hits in tests/). The contract it asserted
512 # was: extra keys preserved, missing keys defaulted. In-place mutation is
513 # safe — the formatter returned a fresh list of fresh dicts.
514 for log in logs:
515 log.setdefault("time", "")
516 log.setdefault("message", "No message")
517 log.setdefault("type", "info")
519 return {"status": "success", "logs": logs}
522@router.get("/log_count/{research_id}")
523def get_log_count(
524 request: Request,
525 research_id,
526 username: Annotated[str, Depends(require_auth)],
527):
528 """Get the total number of logs for a specific research ID"""
529 # Verify ownership before exposing the count.
530 with get_user_db_session(username) as db_session:
531 research = (
532 db_session.query(ResearchHistory.id)
533 .filter_by(id=research_id)
534 .first()
535 )
536 if not research:
537 # Use the shared helper rather than an inline 404, matching the
538 # two sibling call sites in this file (lines ~165 and ~226) and
539 # main's #5386 fix. It emits status/error/message together, a
540 # superset of both historical 404 shapes, so every frontend
541 # reader keeps working.
542 return _research_not_found(research_id)
544 total_logs = get_total_logs_for_research(research_id, username)
546 return {"status": "success", "total_logs": total_logs}