Coverage for src/local_deep_research/web/routers/benchmark.py: 93%

1"""Flask routes for benchmark web interface.""" 

2 

3from fastapi import APIRouter, Depends, Request 

4from fastapi.responses import JSONResponse 

5from ..dependencies.auth import require_auth 

6from ..dependencies.threadpool import run_db_sync 

7from ..dependencies.rate_limit import limiter 

8from ..template_config import templates 

9 

10import time 

11 

12from loguru import logger 

13 

14from ...database.session_context import get_user_db_session 

15from ...llm.providers.base import normalize_provider 

16 

17from local_deep_research.settings import SettingsManager 

18 

19from ...benchmarks.web_api.benchmark_service import benchmark_service 

20from ..dependencies.json_body import read_json_dict 

21from typing import Annotated 

22 

23# Benchmark runs spawn full LLM + search loops (expensive enough that 

24# rate-limiting them is a DoS defense, not just good manners). Tight 

25# cap since a single run can cost tens of LLM completions. 

26_BENCHMARK_START_RATE_LIMIT = "3 per minute" 

27 

28# SHARED bucket across both start endpoints. Applying `limiter.limit()` to 

29# each route separately gave /api/start and /api/start-simple their OWN 

30# 3/min bucket -- 6 starts a minute combined, twice the cap the comment 

31# above describes, for two routes that kick off the same expensive work. 

32# This is the same mistake metrics.py documents fixing for the journal 

33# endpoints (three routes, three 60/min buckets, 180/min combined). 

34# key_func is deliberately omitted: slowapi does `key_func or self._key_func`, 

35# so this keeps the limiter's default per-IP keying rather than quietly 

36# switching these routes to per-user as well. 

37_benchmark_start_limit = limiter.shared_limit( 

38 _BENCHMARK_START_RATE_LIMIT, scope="benchmark_start" 

39) 

40 

41# Create the router for benchmark routes 

42router = APIRouter(prefix="/benchmark", tags=["benchmark"]) 

43 

44# NOTE: Routes use username (not .get()) intentionally. 

45# Depends(require_auth) guarantees the key exists; direct access fails 

46# fast if the dependency is ever removed. 

47 

48 

49def _validate_datasets_config(datasets_config): 

50 """Return ``(total_examples, error)`` for a benchmark dataset mapping. 

51 

52 Dataset counts feed both database totals and ``load_dataset``'s integer 

53 ``num_examples`` argument. Validate the nested shape once at the HTTP 

54 boundary so a scalar entry, a truthy boolean, a fractional value, or a 

55 negative count cannot turn into a late 500 after settings have already 

56 been loaded or a benchmark row has been created. Counts intentionally 

57 have no upper cap (#4080); zero and a missing count remain valid for a 

58 disabled dataset, provided another dataset has a positive count. 

59 """ 

60 if not isinstance(datasets_config, dict): 

61 return 0, "datasets_config must be an object" 

62 

63 total_examples = 0 

64 for config in datasets_config.values(): 

65 if not isinstance(config, dict): 

66 return 0, "Each datasets_config entry must be an object" 

67 

68 count = config.get("count", 0) 

69 if isinstance(count, bool) or not isinstance(count, int) or count < 0: 

70 return 0, "Dataset counts must be non-negative integers" 

71 total_examples += count 

72 

73 return total_examples, None 

74 

75 

76@router.get("/") 

77def index(request: Request, username: Annotated[str, Depends(require_auth)]): 

78 """Benchmark dashboard page.""" 

79 

80 with get_user_db_session(username) as db_session: 

81 settings_manager = SettingsManager(db_session) 

82 

83 # Load evaluation settings from database 

84 eval_settings = { 

85 "evaluation_provider": settings_manager.get_setting( 

86 "benchmark.evaluation.provider", "openai_endpoint" 

87 ), 

88 "evaluation_model": settings_manager.get_setting( 

89 "benchmark.evaluation.model", "" 

90 ), 

91 "evaluation_endpoint_url": settings_manager.get_setting( 

92 "benchmark.evaluation.endpoint_url", "" 

93 ), 

94 "evaluation_temperature": settings_manager.get_setting( 

95 "benchmark.evaluation.temperature", 0 

96 ), 

97 } 

98 

99 return templates.TemplateResponse( 

100 request=request, 

101 name="pages/benchmark.html", 

102 context={"request": request, "eval_settings": eval_settings}, 

103 ) 

104 

105 

106@router.get("/results") 

107def results(request: Request, username: Annotated[str, Depends(require_auth)]): 

108 """Benchmark results history page.""" 

109 return templates.TemplateResponse( 

110 request=request, 

111 name="pages/benchmark_results.html", 

112 context={"request": request}, 

113 ) 

114 

115 

116@router.post("/api/start") 

117@_benchmark_start_limit 

118async def start_benchmark( 

119 request: Request, username: Annotated[str, Depends(require_auth)] 

120): 

121 """Start a new benchmark run.""" 

122 data, body_error = await read_json_dict( 

123 request, "simple", "No data provided" 

124 ) 

125 if body_error is not None: 

126 return body_error 

127 session_id = request.session.get("session_id") 

128 return await run_db_sync(_start_benchmark_sync, data, username, session_id) 

129 

130 

131def _start_benchmark_sync(data, username, session_id): 

132 try: 

133 # Extract configuration 

134 run_name = data.get("run_name") 

135 datasets_config = data.get("datasets_config", {}) 

136 total_examples, datasets_error = _validate_datasets_config( 

137 datasets_config 

138 ) 

139 if datasets_error is not None: 

140 return JSONResponse({"error": datasets_error}, status_code=400) 

141 if total_examples == 0: 

142 return JSONResponse( 

143 { 

144 "error": "At least one dataset with count > 0 must be specified" 

145 }, 

146 status_code=400, 

147 ) 

148 

149 # Get search config from database instead of request 

150 from ...database.session_context import get_user_db_session 

151 from local_deep_research.settings import SettingsManager 

152 

153 # Try to get password from session store for background thread 

154 from ...database.session_passwords import session_password_store 

155 

156 user_password = None 

157 if session_id: 157 ↛ 158line 157 didn't jump to line 158 because the condition on line 157 was never true

158 user_password = session_password_store.get_session_password( 

159 username, session_id 

160 ) 

161 

162 search_config = {} 

163 evaluation_config = {} 

164 

165 with get_user_db_session(username) as db_session: 

166 # Use the logged-in user's settings 

167 settings_manager = SettingsManager(db_session) 

168 

169 # Build search config from database settings 

170 search_config = { 

171 "iterations": int( 

172 settings_manager.get_setting("search.iterations", 8) 

173 ), 

174 "questions_per_iteration": int( 

175 settings_manager.get_setting( 

176 "search.questions_per_iteration", 5 

177 ) 

178 ), 

179 "search_tool": settings_manager.get_setting( 

180 "search.tool", "searxng" 

181 ), 

182 "search_strategy": settings_manager.get_setting( 

183 "search.search_strategy", "focused_iteration" 

184 ), 

185 "model_name": settings_manager.get_setting("llm.model"), 

186 "provider": settings_manager.get_setting("llm.provider"), 

187 "temperature": float( 

188 settings_manager.get_setting("llm.temperature", 0.7) 

189 ), 

190 "max_tokens": settings_manager.get_setting( 

191 "llm.max_tokens", 30000 

192 ), 

193 "context_window_unrestricted": settings_manager.get_setting( 

194 "llm.context_window_unrestricted", True 

195 ), 

196 "context_window_size": settings_manager.get_setting( 

197 "llm.context_window_size", 128000 

198 ), 

199 "local_context_window_size": settings_manager.get_setting( 

200 "llm.local_context_window_size", 8192 

201 ), 

202 } 

203 

204 # Add provider-specific settings 

205 provider = normalize_provider(search_config.get("provider")) 

206 if provider == "openai_endpoint": 

207 search_config["openai_endpoint_url"] = ( 

208 settings_manager.get_setting("llm.openai_endpoint.url") 

209 ) 

210 search_config["openai_endpoint_api_key"] = ( 

211 settings_manager.get_setting("llm.openai_endpoint.api_key") 

212 ) 

213 elif provider == "openai": 

214 search_config["openai_api_key"] = settings_manager.get_setting( 

215 "llm.openai.api_key" 

216 ) 

217 elif provider == "anthropic": 

218 search_config["anthropic_api_key"] = ( 

219 settings_manager.get_setting("llm.anthropic.api_key") 

220 ) 

221 

222 # Get evaluation config from database settings or request. 

223 # When caller supplies one, validate structure + whitelist fields 

224 # — the old code passed the raw dict straight into 

225 # `create_benchmark_run`, where it becomes LLM API call 

226 # parameters (provider name, model name, endpoint URL, API key). 

227 # Without validation a user could override the endpoint to 

228 # point at an arbitrary internal service. 

229 if "evaluation_config" in data: 

230 raw = data["evaluation_config"] 

231 if not isinstance(raw, dict): 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true

232 return JSONResponse( 

233 { 

234 "success": False, 

235 "error": "evaluation_config must be an object", 

236 }, 

237 status_code=400, 

238 ) 

239 _ALLOWED_EVAL_KEYS = { 

240 "provider", 

241 "model_name", 

242 "temperature", 

243 "max_tokens", 

244 } 

245 unknown = set(raw.keys()) - _ALLOWED_EVAL_KEYS 

246 if unknown: 246 ↛ 247line 246 didn't jump to line 247 because the condition on line 246 was never true

247 return JSONResponse( 

248 { 

249 "success": False, 

250 "error": f"Unknown evaluation_config keys: {sorted(unknown)}", 

251 }, 

252 status_code=400, 

253 ) 

254 # Coerce and constrain field types 

255 evaluation_config = {} 

256 if "provider" in raw and isinstance(raw["provider"], str): 256 ↛ 258line 256 didn't jump to line 258 because the condition on line 256 was always true

257 evaluation_config["provider"] = raw["provider"][:64] 

258 if "model_name" in raw and isinstance(raw["model_name"], str): 258 ↛ 260line 258 didn't jump to line 260 because the condition on line 258 was always true

259 evaluation_config["model_name"] = raw["model_name"][:256] 

260 if "temperature" in raw: 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true

261 try: 

262 evaluation_config["temperature"] = max( 

263 0.0, min(float(raw["temperature"]), 2.0) 

264 ) 

265 except (TypeError, ValueError): 

266 pass 

267 if "max_tokens" in raw: 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true

268 try: 

269 evaluation_config["max_tokens"] = max( 

270 1, min(int(raw["max_tokens"]), 100000) 

271 ) 

272 except (TypeError, ValueError): 

273 pass 

274 else: 

275 # Read evaluation config from database settings 

276 evaluation_provider = normalize_provider( 

277 settings_manager.get_setting( 

278 "benchmark.evaluation.provider", "openai_endpoint" 

279 ) 

280 ) 

281 evaluation_model = settings_manager.get_setting( 

282 "benchmark.evaluation.model", "anthropic/claude-3.7-sonnet" 

283 ) 

284 evaluation_temperature = float( 

285 settings_manager.get_setting( 

286 "benchmark.evaluation.temperature", 0 

287 ) 

288 ) 

289 

290 evaluation_config = { 

291 "provider": evaluation_provider, 

292 "model_name": evaluation_model, 

293 "temperature": evaluation_temperature, 

294 } 

295 

296 # Add provider-specific settings for evaluation 

297 if evaluation_provider == "openai_endpoint": 

298 evaluation_config["openai_endpoint_url"] = ( 

299 settings_manager.get_setting( 

300 "benchmark.evaluation.endpoint_url", 

301 "https://openrouter.ai/api/v1", 

302 ) 

303 ) 

304 evaluation_config["openai_endpoint_api_key"] = ( 

305 settings_manager.get_setting( 

306 "llm.openai_endpoint.api_key" 

307 ) 

308 ) 

309 elif evaluation_provider == "openai": 

310 evaluation_config["openai_api_key"] = ( 

311 settings_manager.get_setting("llm.openai.api_key") 

312 ) 

313 elif evaluation_provider == "anthropic": 313 ↛ 319line 313 didn't jump to line 319

314 evaluation_config["anthropic_api_key"] = ( 

315 settings_manager.get_setting("llm.anthropic.api_key") 

316 ) 

317 

318 # Create benchmark run 

319 benchmark_run_id = benchmark_service.create_benchmark_run( 

320 run_name=run_name, 

321 search_config=search_config, 

322 evaluation_config=evaluation_config, 

323 datasets_config=datasets_config, 

324 username=username, 

325 user_password=user_password, 

326 ) 

327 

328 # Start benchmark 

329 success = benchmark_service.start_benchmark( 

330 benchmark_run_id, username, user_password 

331 ) 

332 

333 if success: 

334 return { 

335 "success": True, 

336 "benchmark_run_id": benchmark_run_id, 

337 "message": "Benchmark started successfully", 

338 } 

339 

340 return JSONResponse( 

341 {"success": False, "error": "Failed to start benchmark"}, 

342 status_code=500, 

343 ) 

344 

345 except Exception: 

346 logger.exception("Error starting benchmark") 

347 return JSONResponse( 

348 {"success": False, "error": "An internal error has occurred."}, 

349 status_code=500, 

350 ) 

351 

352 

353@router.get("/api/running") 

354def get_running_benchmark( 

355 request: Request, username: Annotated[str, Depends(require_auth)] 

356): 

357 """Check if there's a running benchmark and return its ID.""" 

358 try: 

359 from ...database.models.benchmark import BenchmarkRun, BenchmarkStatus 

360 from ...database.session_context import get_user_db_session 

361 

362 with get_user_db_session(username) as session: 

363 # Find any benchmark that's currently running 

364 running_benchmark = ( 

365 session.query(BenchmarkRun) 

366 .filter(BenchmarkRun.status == BenchmarkStatus.IN_PROGRESS) 

367 .order_by(BenchmarkRun.created_at.desc()) 

368 .first() 

369 ) 

370 

371 if running_benchmark: 

372 return { 

373 "success": True, 

374 "benchmark_run_id": running_benchmark.id, 

375 "run_name": running_benchmark.run_name, 

376 "total_examples": running_benchmark.total_examples, 

377 "completed_examples": running_benchmark.completed_examples, 

378 } 

379 

380 return { 

381 "success": False, 

382 "message": "No running benchmark found", 

383 } 

384 

385 except Exception: 

386 logger.exception("Error checking for running benchmark") 

387 return JSONResponse( 

388 {"success": False, "error": "An internal error has occurred."}, 

389 status_code=500, 

390 ) 

391 

392 

393@router.get("/api/status/{benchmark_run_id}") 

394@limiter.exempt 

395def get_benchmark_status( 

396 request: Request, 

397 benchmark_run_id: int, 

398 username: Annotated[str, Depends(require_auth)], 

399): 

400 """Get status of a benchmark run.""" 

401 try: 

402 status = benchmark_service.get_benchmark_status( 

403 benchmark_run_id, username 

404 ) 

405 

406 if status: 

407 logger.info( 

408 f"Returning status for benchmark {benchmark_run_id}: " 

409 f"completed={status.get('completed_examples')}, " 

410 f"overall_acc={status.get('overall_accuracy')}, " 

411 f"avg_time={status.get('avg_time_per_example')}, " 

412 f"estimated_remaining={status.get('estimated_time_remaining')}" 

413 ) 

414 return {"success": True, "status": status} 

415 return JSONResponse( 

416 {"success": False, "error": "Benchmark run not found"}, 

417 status_code=404, 

418 ) 

419 

420 except Exception: 

421 logger.exception("Error getting benchmark status") 

422 return JSONResponse( 

423 {"success": False, "error": "An internal error has occurred."}, 

424 status_code=500, 

425 ) 

426 

427 

428@router.post("/api/cancel/{benchmark_run_id}") 

429def cancel_benchmark( 

430 request: Request, 

431 benchmark_run_id: int, 

432 username: Annotated[str, Depends(require_auth)], 

433): 

434 """Cancel a running benchmark.""" 

435 try: 

436 success = benchmark_service.cancel_benchmark(benchmark_run_id, username) 

437 

438 if success: 

439 return { 

440 "success": True, 

441 "message": "Benchmark cancelled successfully", 

442 } 

443 

444 return JSONResponse( 

445 {"success": False, "error": "Failed to cancel benchmark"}, 

446 status_code=500, 

447 ) 

448 

449 except Exception: 

450 logger.exception("Error cancelling benchmark") 

451 return JSONResponse( 

452 {"success": False, "error": "An internal error has occurred."}, 

453 status_code=500, 

454 ) 

455 

456 

457@router.get("/api/history") 

458def get_benchmark_history( 

459 request: Request, username: Annotated[str, Depends(require_auth)] 

460): 

461 """Get list of recent benchmark runs.""" 

462 try: 

463 from ...database.models.benchmark import BenchmarkRun 

464 from ...database.session_context import get_user_db_session 

465 

466 with get_user_db_session(username) as session: 

467 # Get all benchmark runs (completed, failed, cancelled, or in-progress) 

468 runs = ( 

469 session.query(BenchmarkRun) 

470 .order_by(BenchmarkRun.created_at.desc()) 

471 .limit(50) 

472 .all() 

473 ) 

474 

475 # Format runs for display 

476 formatted_runs = [] 

477 for run in runs: 

478 # Calculate average processing time from results 

479 avg_processing_time = None 

480 avg_search_results = None 

481 try: 

482 from sqlalchemy import func 

483 

484 from ...database.models.benchmark import BenchmarkResult 

485 

486 avg_result = ( 

487 session.query(func.avg(BenchmarkResult.processing_time)) 

488 .filter( 

489 BenchmarkResult.benchmark_run_id == run.id, 

490 BenchmarkResult.processing_time.isnot(None), 

491 BenchmarkResult.processing_time > 0, 

492 ) 

493 .scalar() 

494 ) 

495 

496 if avg_result: 

497 avg_processing_time = float(avg_result) 

498 except Exception: 

499 logger.warning( 

500 f"Error calculating avg processing time for run {run.id}" 

501 ) 

502 

503 # Calculate average search results and total search requests from metrics 

504 total_search_requests = None 

505 try: 

506 from ...database.models import SearchCall 

507 from ...metrics.search_tracker import ( 

508 SEARCH_STATUS_SKIPPED, 

509 ) 

510 

511 # Get all results for this run to find research_ids 

512 results = ( 

513 session.query(BenchmarkResult) 

514 .filter(BenchmarkResult.benchmark_run_id == run.id) 

515 .all() 

516 ) 

517 

518 research_ids = [ 

519 r.research_id for r in results if r.research_id 

520 ] 

521 

522 if research_ids: 

523 # SearchCall is in the same per-user DB, query directly 

524 search_calls = ( 

525 session.query(SearchCall) 

526 .filter(SearchCall.research_id.in_(research_ids)) 

527 .all() 

528 ) 

529 

530 # Group by research_id and calculate metrics per research session 

531 research_results = {} 

532 research_requests = {} 

533 

534 for call in search_calls: 

535 # A search skipped because its engine was 

536 # unavailable sent no request. 

537 if call.success_status == SEARCH_STATUS_SKIPPED: 

538 continue 

539 if call.research_id: 539 ↛ 534line 539 didn't jump to line 534 because the condition on line 539 was always true

540 if call.research_id not in research_results: 

541 research_results[call.research_id] = 0 

542 research_requests[call.research_id] = 0 

543 research_results[call.research_id] += ( 

544 call.results_count or 0 

545 ) 

546 research_requests[call.research_id] += 1 

547 

548 # Calculate averages across research sessions 

549 if research_results: 549 ↛ 565line 549 didn't jump to line 565 because the condition on line 549 was always true

550 total_results = sum(research_results.values()) 

551 avg_search_results = total_results / len( 

552 research_results 

553 ) 

554 

555 total_requests = sum(research_requests.values()) 

556 total_search_requests = total_requests / len( 

557 research_requests 

558 ) 

559 

560 except Exception: 

561 logger.warning( 

562 f"Error calculating search metrics for run {run.id}" 

563 ) 

564 

565 formatted_runs.append( 

566 { 

567 "id": run.id, 

568 "run_name": run.run_name or f"Benchmark #{run.id}", 

569 "created_at": run.created_at.isoformat(), 

570 # `start_time` is the wall-clock instant work began 

571 # (status flipped to IN_PROGRESS); used by the YAML 

572 # download for `date_tested` instead of the (broken) 

573 # download-time stamp. NULL on rows that never started. 

574 "start_time": run.start_time.isoformat() 

575 if run.start_time 

576 else None, 

577 # `ldr_version` is captured by start_benchmark; NULL 

578 # on rows created before migration 0014. The YAML 

579 # download substitutes "unknown (pre-0014 run)". 

580 # `settings_snapshot` is intentionally NOT included 

581 # here — it's ~184KB per run and would balloon the 

582 # history list response. Loaded on-demand from 

583 # /api/results/<id>/export when YAML is downloaded. 

584 "ldr_version": run.ldr_version, 

585 "total_examples": run.total_examples, 

586 "completed_examples": run.completed_examples, 

587 "overall_accuracy": run.overall_accuracy, 

588 "status": run.status.value, 

589 "search_config": run.search_config, 

590 "evaluation_config": run.evaluation_config, 

591 "datasets_config": run.datasets_config, 

592 "avg_processing_time": avg_processing_time, 

593 "avg_search_results": avg_search_results, 

594 "total_search_requests": total_search_requests, 

595 } 

596 ) 

597 

598 return {"success": True, "runs": formatted_runs} 

599 

600 except Exception: 

601 logger.exception("Error getting benchmark history") 

602 return JSONResponse( 

603 {"success": False, "error": "An internal error has occurred."}, 

604 status_code=500, 

605 ) 

606 

607 

608@router.get("/api/results/{benchmark_run_id}") 

609@limiter.exempt 

610def get_benchmark_results( 

611 request: Request, 

612 benchmark_run_id: int, 

613 username: Annotated[str, Depends(require_auth)], 

614): 

615 """Get detailed results for a benchmark run.""" 

616 try: 

617 from ...database.models.benchmark import BenchmarkResult 

618 from ...database.session_context import get_user_db_session 

619 

620 logger.info(f"Getting results for benchmark {benchmark_run_id}") 

621 

622 # First sync any pending results from active runs 

623 benchmark_service.sync_pending_results(benchmark_run_id, username) 

624 persistence_error = benchmark_service.get_result_persistence_error( 

625 benchmark_run_id, username 

626 ) 

627 with get_user_db_session(username) as session: 

628 # Get recent results (limit to last 10) 

629 try: 

630 limit = int(request.query_params.get("limit", 10)) 

631 except (TypeError, ValueError): 

632 limit = 10 

633 limit = max(1, min(limit, 1000)) 

634 

635 results = ( 

636 session.query(BenchmarkResult) 

637 .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id) 

638 # Results are returned unfiltered, including rows whose 

639 # evaluation is still pending — same as main's equivalent 

640 # route (benchmarks/web_api/benchmark_routes.py), which also 

641 # has no is_correct filter here. The is_correct.isnot(None) 

642 # filter belongs to the aggregate/stats queries in 

643 # benchmark_service.py, not to this listing. 

644 .order_by(BenchmarkResult.id.desc()) # Most recent first 

645 .limit(limit) 

646 .all() 

647 ) 

648 

649 logger.info(f"Found {len(results)} results") 

650 

651 # Build a map of research_id to total search results 

652 search_results_by_research_id = {} 

653 try: 

654 from ...database.models import SearchCall 

655 

656 # Get all unique research_ids from our results 

657 research_ids = [r.research_id for r in results if r.research_id] 

658 

659 if research_ids: 

660 # SearchCall is in the same per-user DB, query directly 

661 all_search_calls = ( 

662 session.query(SearchCall) 

663 .filter(SearchCall.research_id.in_(research_ids)) 

664 .all() 

665 ) 

666 

667 # Group search results by research_id 

668 for call in all_search_calls: 

669 if call.research_id: 669 ↛ 668line 669 didn't jump to line 668 because the condition on line 669 was always true

670 if ( 670 ↛ 677line 670 didn't jump to line 677 because the condition on line 670 was always true

671 call.research_id 

672 not in search_results_by_research_id 

673 ): 

674 search_results_by_research_id[ 

675 call.research_id 

676 ] = 0 

677 search_results_by_research_id[call.research_id] += ( 

678 call.results_count or 0 

679 ) 

680 

681 logger.info( 

682 f"Found search metrics for {len(search_results_by_research_id)} research IDs from {len(all_search_calls)} total search calls" 

683 ) 

684 logger.debug( 

685 f"Research IDs from results: {research_ids[:5] if len(research_ids) > 5 else research_ids}" 

686 ) 

687 logger.debug( 

688 f"Search results by research_id: {dict(list(search_results_by_research_id.items())[:5])}" 

689 ) 

690 except Exception: 

691 logger.exception( 

692 f"Error getting search metrics for benchmark {benchmark_run_id}" 

693 ) 

694 

695 # Format results for UI display 

696 formatted_results = [] 

697 for result in results: 

698 # Get search result count using research_id 

699 search_result_count = 0 

700 

701 try: 

702 if ( 

703 result.research_id 

704 and result.research_id in search_results_by_research_id 

705 ): 

706 search_result_count = search_results_by_research_id[ 

707 result.research_id 

708 ] 

709 logger.debug( 

710 f"Found {search_result_count} search results for research_id {result.research_id}" 

711 ) 

712 

713 except Exception: 

714 logger.exception( 

715 f"Error getting search results for result {result.example_id}" 

716 ) 

717 

718 formatted_results.append( 

719 { 

720 "example_id": result.example_id, 

721 "dataset_type": result.dataset_type.value, 

722 "question": result.question, 

723 "correct_answer": result.correct_answer, 

724 "model_answer": result.extracted_answer, 

725 "full_response": result.response, 

726 "is_correct": result.is_correct, 

727 "confidence": result.confidence, 

728 "grader_response": result.grader_response, 

729 "processing_time": result.processing_time, 

730 "search_result_count": search_result_count, 

731 "sources": result.sources, 

732 "completed_at": result.completed_at.isoformat() 

733 if result.completed_at 

734 else None, 

735 } 

736 ) 

737 

738 payload = {"success": True, "results": formatted_results} 

739 if persistence_error: 

740 payload["persistence_error"] = persistence_error 

741 return payload 

742 

743 except Exception: 

744 logger.exception("Error getting benchmark results") 

745 return JSONResponse( 

746 {"success": False, "error": "An internal error has occurred."}, 

747 status_code=500, 

748 ) 

749 

750 

751@router.get("/api/results/{benchmark_run_id}/export") 

752def export_benchmark_results( 

753 request: Request, 

754 benchmark_run_id: int, 

755 username: Annotated[str, Depends(require_auth)], 

756): 

757 """Get lightweight results for YAML export plus run-level provenance. 

758 

759 Returns a `metadata` block alongside the per-result rows so the YAML 

760 download path can stamp the version/settings that ran the benchmark 

761 instead of falling back to download-time values. Pre-0014 rows return 

762 NULL for the new metadata fields. The `success: True` envelope is 

763 preserved so the existing JS contract (5 callsites in 

764 benchmark_results.html) keeps working. 

765 """ 

766 try: 

767 from sqlalchemy.orm import load_only 

768 

769 from ...database.models.benchmark import ( 

770 BenchmarkResult, 

771 BenchmarkRun, 

772 ) 

773 from ...database.session_context import get_user_db_session 

774 

775 # The ~184KB settings snapshot is only loaded/returned when the 

776 # client opts in (Export -> "Include settings snapshot"); the 

777 # default summary download skips it to avoid transferring a blob it 

778 # would discard. 

779 include_settings = request.query_params.get( 

780 "include_settings", "" 

781 ).lower() in ( 

782 "1", 

783 "true", 

784 ) 

785 logger.info( 

786 "Exporting benchmark results for run {} by user {}", 

787 benchmark_run_id, 

788 username, 

789 ) 

790 with get_user_db_session(username) as session: 

791 # Fetch only the provenance columns we need from BenchmarkRun. 

792 # `load_only` keeps us off the heavy JSON config blobs (and the 

793 # settings_snapshot blob unless it was requested). 

794 run_columns = [ 

795 BenchmarkRun.ldr_version, 

796 BenchmarkRun.start_time, 

797 BenchmarkRun.created_at, 

798 ] 

799 if include_settings: 

800 run_columns.append(BenchmarkRun.settings_snapshot) 

801 run = ( 

802 session.query(BenchmarkRun) 

803 .options(load_only(*run_columns)) 

804 .filter(BenchmarkRun.id == benchmark_run_id) 

805 .one_or_none() 

806 ) 

807 

808 results = ( 

809 session.query(BenchmarkResult) 

810 .options( 

811 load_only( 

812 BenchmarkResult.example_id, 

813 BenchmarkResult.dataset_type, 

814 BenchmarkResult.question, 

815 BenchmarkResult.correct_answer, 

816 BenchmarkResult.extracted_answer, 

817 BenchmarkResult.is_correct, 

818 BenchmarkResult.confidence, 

819 BenchmarkResult.processing_time, 

820 BenchmarkResult.completed_at, 

821 ) 

822 ) 

823 .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id) 

824 .order_by(BenchmarkResult.id.asc()) 

825 .all() 

826 ) 

827 

828 formatted = [] 

829 for r in results: 

830 formatted.append( 

831 { 

832 "example_id": r.example_id, 

833 "dataset_type": r.dataset_type.value, 

834 "question": r.question, 

835 "correct_answer": r.correct_answer, 

836 "model_answer": r.extracted_answer, 

837 "is_correct": r.is_correct, 

838 "confidence": r.confidence, 

839 "processing_time": r.processing_time, 

840 "completed_at": r.completed_at.isoformat() 

841 if r.completed_at 

842 else None, 

843 } 

844 ) 

845 

846 # Build metadata block. Each field is independently null-checked 

847 # so a pre-0014 row, or an in-flight run with no start_time, 

848 # serializes cleanly as JSON null. 

849 if run is not None: 

850 started_at = ( 

851 run.start_time.isoformat() 

852 if run.start_time 

853 else ( 

854 run.created_at.isoformat() if run.created_at else None 

855 ) 

856 ) 

857 metadata = { 

858 "ldr_version": run.ldr_version, 

859 "started_at": started_at, 

860 "settings_snapshot": ( 

861 run.settings_snapshot if include_settings else None 

862 ), 

863 } 

864 else: 

865 metadata = { 

866 "ldr_version": None, 

867 "started_at": None, 

868 "settings_snapshot": None, 

869 } 

870 

871 logger.info( 

872 "Exported {} results for benchmark run {}", 

873 len(formatted), 

874 benchmark_run_id, 

875 ) 

876 return { 

877 "success": True, 

878 "metadata": metadata, 

879 "results": formatted, 

880 } 

881 

882 except Exception: 

883 logger.exception("Error exporting benchmark results") 

884 return JSONResponse( 

885 {"success": False, "error": "An internal error has occurred."}, 

886 status_code=500, 

887 ) 

888 

889 

890@router.get("/api/configs") 

891def get_saved_configs( 

892 request: Request, username: Annotated[str, Depends(require_auth)] 

893): 

894 """Get list of saved benchmark configurations.""" 

895 try: 

896 # Returns built-in default configs (saved configs not yet in DB) 

897 default_configs = [ 

898 { 

899 "id": 1, 

900 "name": "Quick Test", 

901 "description": "Fast benchmark with minimal examples", 

902 "search_config": { 

903 "iterations": 3, 

904 "questions_per_iteration": 3, 

905 "search_tool": "searxng", 

906 "search_strategy": "focused_iteration", 

907 },