Coverage for src/local_deep_research/web/routers/benchmark.py: 93%

1"""Flask routes for benchmark web interface.""" 

2 

3from fastapi import APIRouter, Depends, Request 

4from fastapi.responses import JSONResponse 

5from ..dependencies.auth import require_auth 

6from ..dependencies.threadpool import run_db_sync 

7from ..dependencies.rate_limit import limiter 

8from ..template_config import templates 

9 

10import time 

11 

12from loguru import logger 

13 

14from ...database.session_context import get_user_db_session 

15from ...llm.providers.base import normalize_provider 

16 

17from local_deep_research.settings import SettingsManager 

18 

19from ...benchmarks.web_api.benchmark_service import benchmark_service 

20from ..dependencies.json_body import json_body_error 

21from typing import Annotated 

22 

23# Benchmark runs spawn full LLM + search loops (expensive enough that 

24# rate-limiting them is a DoS defense, not just good manners). Tight 

25# cap since a single run can cost tens of LLM completions. 

26_BENCHMARK_START_RATE_LIMIT = "3 per minute" 

27 

28# SHARED bucket across both start endpoints. Applying `limiter.limit()` to 

29# each route separately gave /api/start and /api/start-simple their OWN 

30# 3/min bucket -- 6 starts a minute combined, twice the cap the comment 

31# above describes, for two routes that kick off the same expensive work. 

32# This is the same mistake metrics.py documents fixing for the journal 

33# endpoints (three routes, three 60/min buckets, 180/min combined). 

34# key_func is deliberately omitted: slowapi does `key_func or self._key_func`, 

35# so this keeps the limiter's default per-IP keying rather than quietly 

36# switching these routes to per-user as well. 

37_benchmark_start_limit = limiter.shared_limit( 

38 _BENCHMARK_START_RATE_LIMIT, scope="benchmark_start" 

39) 

40 

41# Create the router for benchmark routes 

42router = APIRouter(prefix="/benchmark", tags=["benchmark"]) 

43 

44# NOTE: Routes use username (not .get()) intentionally. 

45# Depends(require_auth) guarantees the key exists; direct access fails 

46# fast if the dependency is ever removed. 

47 

48 

49def _validate_datasets_config(datasets_config): 

50 """Return ``(total_examples, error)`` for a benchmark dataset mapping. 

51 

52 Dataset counts feed both database totals and ``load_dataset``'s integer 

53 ``num_examples`` argument. Validate the nested shape once at the HTTP 

54 boundary so a scalar entry, a truthy boolean, a fractional value, or a 

55 negative count cannot turn into a late 500 after settings have already 

56 been loaded or a benchmark row has been created. Counts intentionally 

57 have no upper cap (#4080); zero and a missing count remain valid for a 

58 disabled dataset, provided another dataset has a positive count. 

59 """ 

60 if not isinstance(datasets_config, dict): 

61 return 0, "datasets_config must be an object" 

62 

63 total_examples = 0 

64 for config in datasets_config.values(): 

65 if not isinstance(config, dict): 

66 return 0, "Each datasets_config entry must be an object" 

67 

68 count = config.get("count", 0) 

69 if isinstance(count, bool) or not isinstance(count, int) or count < 0: 

70 return 0, "Dataset counts must be non-negative integers" 

71 total_examples += count 

72 

73 return total_examples, None 

74 

75 

76@router.get("/") 

77def index(request: Request, username: Annotated[str, Depends(require_auth)]): 

78 """Benchmark dashboard page.""" 

79 

80 with get_user_db_session(username) as db_session: 

81 settings_manager = SettingsManager(db_session) 

82 

83 # Load evaluation settings from database 

84 eval_settings = { 

85 "evaluation_provider": settings_manager.get_setting( 

86 "benchmark.evaluation.provider", "openai_endpoint" 

87 ), 

88 "evaluation_model": settings_manager.get_setting( 

89 "benchmark.evaluation.model", "" 

90 ), 

91 "evaluation_endpoint_url": settings_manager.get_setting( 

92 "benchmark.evaluation.endpoint_url", "" 

93 ), 

94 "evaluation_temperature": settings_manager.get_setting( 

95 "benchmark.evaluation.temperature", 0 

96 ), 

97 } 

98 

99 return templates.TemplateResponse( 

100 request=request, 

101 name="pages/benchmark.html", 

102 context={"request": request, "eval_settings": eval_settings}, 

103 ) 

104 

105 

106@router.get("/results") 

107def results(request: Request, username: Annotated[str, Depends(require_auth)]): 

108 """Benchmark results history page.""" 

109 return templates.TemplateResponse( 

110 request=request, 

111 name="pages/benchmark_results.html", 

112 context={"request": request}, 

113 ) 

114 

115 

116@router.post("/api/start") 

117@_benchmark_start_limit 

118async def start_benchmark( 

119 request: Request, username: Annotated[str, Depends(require_auth)] 

120): 

121 """Start a new benchmark run.""" 

122 data = await request.json() 

123 if not isinstance(data, dict): 

124 return json_body_error("simple", "No data provided") 

125 session_id = request.session.get("session_id") 

126 return await run_db_sync(_start_benchmark_sync, data, username, session_id) 

127 

128 

129def _start_benchmark_sync(data, username, session_id): 

130 try: 

131 # Extract configuration 

132 run_name = data.get("run_name") 

133 datasets_config = data.get("datasets_config", {}) 

134 total_examples, datasets_error = _validate_datasets_config( 

135 datasets_config 

136 ) 

137 if datasets_error is not None: 

138 return JSONResponse({"error": datasets_error}, status_code=400) 

139 if total_examples == 0: 

140 return JSONResponse( 

141 { 

142 "error": "At least one dataset with count > 0 must be specified" 

143 }, 

144 status_code=400, 

145 ) 

146 

147 # Get search config from database instead of request 

148 from ...database.session_context import get_user_db_session 

149 from local_deep_research.settings import SettingsManager 

150 

151 # Try to get password from session store for background thread 

152 from ...database.session_passwords import session_password_store 

153 

154 user_password = None 

155 if session_id: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true

156 user_password = session_password_store.get_session_password( 

157 username, session_id 

158 ) 

159 

160 search_config = {} 

161 evaluation_config = {} 

162 

163 with get_user_db_session(username) as db_session: 

164 # Use the logged-in user's settings 

165 settings_manager = SettingsManager(db_session) 

166 

167 # Build search config from database settings 

168 search_config = { 

169 "iterations": int( 

170 settings_manager.get_setting("search.iterations", 8) 

171 ), 

172 "questions_per_iteration": int( 

173 settings_manager.get_setting( 

174 "search.questions_per_iteration", 5 

175 ) 

176 ), 

177 "search_tool": settings_manager.get_setting( 

178 "search.tool", "searxng" 

179 ), 

180 "search_strategy": settings_manager.get_setting( 

181 "search.search_strategy", "focused_iteration" 

182 ), 

183 "model_name": settings_manager.get_setting("llm.model"), 

184 "provider": settings_manager.get_setting("llm.provider"), 

185 "temperature": float( 

186 settings_manager.get_setting("llm.temperature", 0.7) 

187 ), 

188 "max_tokens": settings_manager.get_setting( 

189 "llm.max_tokens", 30000 

190 ), 

191 "context_window_unrestricted": settings_manager.get_setting( 

192 "llm.context_window_unrestricted", True 

193 ), 

194 "context_window_size": settings_manager.get_setting( 

195 "llm.context_window_size", 128000 

196 ), 

197 "local_context_window_size": settings_manager.get_setting( 

198 "llm.local_context_window_size", 8192 

199 ), 

200 } 

201 

202 # Add provider-specific settings 

203 provider = normalize_provider(search_config.get("provider")) 

204 if provider == "openai_endpoint": 

205 search_config["openai_endpoint_url"] = ( 

206 settings_manager.get_setting("llm.openai_endpoint.url") 

207 ) 

208 search_config["openai_endpoint_api_key"] = ( 

209 settings_manager.get_setting("llm.openai_endpoint.api_key") 

210 ) 

211 elif provider == "openai": 

212 search_config["openai_api_key"] = settings_manager.get_setting( 

213 "llm.openai.api_key" 

214 ) 

215 elif provider == "anthropic": 

216 search_config["anthropic_api_key"] = ( 

217 settings_manager.get_setting("llm.anthropic.api_key") 

218 ) 

219 

220 # Get evaluation config from database settings or request. 

221 # When caller supplies one, validate structure + whitelist fields 

222 # — the old code passed the raw dict straight into 

223 # `create_benchmark_run`, where it becomes LLM API call 

224 # parameters (provider name, model name, endpoint URL, API key). 

225 # Without validation a user could override the endpoint to 

226 # point at an arbitrary internal service. 

227 if "evaluation_config" in data: 

228 raw = data["evaluation_config"] 

229 if not isinstance(raw, dict): 229 ↛ 230line 229 didn't jump to line 230 because the condition on line 229 was never true

230 return JSONResponse( 

231 { 

232 "success": False, 

233 "error": "evaluation_config must be an object", 

234 }, 

235 status_code=400, 

236 ) 

237 _ALLOWED_EVAL_KEYS = { 

238 "provider", 

239 "model_name", 

240 "temperature", 

241 "max_tokens", 

242 } 

243 unknown = set(raw.keys()) - _ALLOWED_EVAL_KEYS 

244 if unknown: 244 ↛ 245line 244 didn't jump to line 245 because the condition on line 244 was never true

245 return JSONResponse( 

246 { 

247 "success": False, 

248 "error": f"Unknown evaluation_config keys: {sorted(unknown)}", 

249 }, 

250 status_code=400, 

251 ) 

252 # Coerce and constrain field types 

253 evaluation_config = {} 

254 if "provider" in raw and isinstance(raw["provider"], str): 254 ↛ 256line 254 didn't jump to line 256 because the condition on line 254 was always true

255 evaluation_config["provider"] = raw["provider"][:64] 

256 if "model_name" in raw and isinstance(raw["model_name"], str): 256 ↛ 258line 256 didn't jump to line 258 because the condition on line 256 was always true

257 evaluation_config["model_name"] = raw["model_name"][:256] 

258 if "temperature" in raw: 258 ↛ 259line 258 didn't jump to line 259 because the condition on line 258 was never true

259 try: 

260 evaluation_config["temperature"] = max( 

261 0.0, min(float(raw["temperature"]), 2.0) 

262 ) 

263 except (TypeError, ValueError): 

264 pass 

265 if "max_tokens" in raw: 265 ↛ 266line 265 didn't jump to line 266 because the condition on line 265 was never true

266 try: 

267 evaluation_config["max_tokens"] = max( 

268 1, min(int(raw["max_tokens"]), 100000) 

269 ) 

270 except (TypeError, ValueError): 

271 pass 

272 else: 

273 # Read evaluation config from database settings 

274 evaluation_provider = normalize_provider( 

275 settings_manager.get_setting( 

276 "benchmark.evaluation.provider", "openai_endpoint" 

277 ) 

278 ) 

279 evaluation_model = settings_manager.get_setting( 

280 "benchmark.evaluation.model", "anthropic/claude-3.7-sonnet" 

281 ) 

282 evaluation_temperature = float( 

283 settings_manager.get_setting( 

284 "benchmark.evaluation.temperature", 0 

285 ) 

286 ) 

287 

288 evaluation_config = { 

289 "provider": evaluation_provider, 

290 "model_name": evaluation_model, 

291 "temperature": evaluation_temperature, 

292 } 

293 

294 # Add provider-specific settings for evaluation 

295 if evaluation_provider == "openai_endpoint": 

296 evaluation_config["openai_endpoint_url"] = ( 

297 settings_manager.get_setting( 

298 "benchmark.evaluation.endpoint_url", 

299 "https://openrouter.ai/api/v1", 

300 ) 

301 ) 

302 evaluation_config["openai_endpoint_api_key"] = ( 

303 settings_manager.get_setting( 

304 "llm.openai_endpoint.api_key" 

305 ) 

306 ) 

307 elif evaluation_provider == "openai": 

308 evaluation_config["openai_api_key"] = ( 

309 settings_manager.get_setting("llm.openai.api_key") 

310 ) 

311 elif evaluation_provider == "anthropic": 311 ↛ 317line 311 didn't jump to line 317

312 evaluation_config["anthropic_api_key"] = ( 

313 settings_manager.get_setting("llm.anthropic.api_key") 

314 ) 

315 

316 # Create benchmark run 

317 benchmark_run_id = benchmark_service.create_benchmark_run( 

318 run_name=run_name, 

319 search_config=search_config, 

320 evaluation_config=evaluation_config, 

321 datasets_config=datasets_config, 

322 username=username, 

323 user_password=user_password, 

324 ) 

325 

326 # Start benchmark 

327 success = benchmark_service.start_benchmark( 

328 benchmark_run_id, username, user_password 

329 ) 

330 

331 if success: 

332 return { 

333 "success": True, 

334 "benchmark_run_id": benchmark_run_id, 

335 "message": "Benchmark started successfully", 

336 } 

337 

338 return JSONResponse( 

339 {"success": False, "error": "Failed to start benchmark"}, 

340 status_code=500, 

341 ) 

342 

343 except Exception: 

344 logger.exception("Error starting benchmark") 

345 return JSONResponse( 

346 {"success": False, "error": "An internal error has occurred."}, 

347 status_code=500, 

348 ) 

349 

350 

351@router.get("/api/running") 

352def get_running_benchmark( 

353 request: Request, username: Annotated[str, Depends(require_auth)] 

354): 

355 """Check if there's a running benchmark and return its ID.""" 

356 try: 

357 from ...database.models.benchmark import BenchmarkRun, BenchmarkStatus 

358 from ...database.session_context import get_user_db_session 

359 

360 with get_user_db_session(username) as session: 

361 # Find any benchmark that's currently running 

362 running_benchmark = ( 

363 session.query(BenchmarkRun) 

364 .filter(BenchmarkRun.status == BenchmarkStatus.IN_PROGRESS) 

365 .order_by(BenchmarkRun.created_at.desc()) 

366 .first() 

367 ) 

368 

369 if running_benchmark: 

370 return { 

371 "success": True, 

372 "benchmark_run_id": running_benchmark.id, 

373 "run_name": running_benchmark.run_name, 

374 "total_examples": running_benchmark.total_examples, 

375 "completed_examples": running_benchmark.completed_examples, 

376 } 

377 

378 return { 

379 "success": False, 

380 "message": "No running benchmark found", 

381 } 

382 

383 except Exception: 

384 logger.exception("Error checking for running benchmark") 

385 return JSONResponse( 

386 {"success": False, "error": "An internal error has occurred."}, 

387 status_code=500, 

388 ) 

389 

390 

391@router.get("/api/status/{benchmark_run_id}") 

392@limiter.exempt 

393def get_benchmark_status( 

394 request: Request, 

395 benchmark_run_id: int, 

396 username: Annotated[str, Depends(require_auth)], 

397): 

398 """Get status of a benchmark run.""" 

399 try: 

400 status = benchmark_service.get_benchmark_status( 

401 benchmark_run_id, username 

402 ) 

403 

404 if status: 

405 logger.info( 

406 f"Returning status for benchmark {benchmark_run_id}: " 

407 f"completed={status.get('completed_examples')}, " 

408 f"overall_acc={status.get('overall_accuracy')}, " 

409 f"avg_time={status.get('avg_time_per_example')}, " 

410 f"estimated_remaining={status.get('estimated_time_remaining')}" 

411 ) 

412 return {"success": True, "status": status} 

413 return JSONResponse( 

414 {"success": False, "error": "Benchmark run not found"}, 

415 status_code=404, 

416 ) 

417 

418 except Exception: 

419 logger.exception("Error getting benchmark status") 

420 return JSONResponse( 

421 {"success": False, "error": "An internal error has occurred."}, 

422 status_code=500, 

423 ) 

424 

425 

426@router.post("/api/cancel/{benchmark_run_id}") 

427def cancel_benchmark( 

428 request: Request, 

429 benchmark_run_id: int, 

430 username: Annotated[str, Depends(require_auth)], 

431): 

432 """Cancel a running benchmark.""" 

433 try: 

434 success = benchmark_service.cancel_benchmark(benchmark_run_id, username) 

435 

436 if success: 

437 return { 

438 "success": True, 

439 "message": "Benchmark cancelled successfully", 

440 } 

441 

442 return JSONResponse( 

443 {"success": False, "error": "Failed to cancel benchmark"}, 

444 status_code=500, 

445 ) 

446 

447 except Exception: 

448 logger.exception("Error cancelling benchmark") 

449 return JSONResponse( 

450 {"success": False, "error": "An internal error has occurred."}, 

451 status_code=500, 

452 ) 

453 

454 

455@router.get("/api/history") 

456def get_benchmark_history( 

457 request: Request, username: Annotated[str, Depends(require_auth)] 

458): 

459 """Get list of recent benchmark runs.""" 

460 try: 

461 from ...database.models.benchmark import BenchmarkRun 

462 from ...database.session_context import get_user_db_session 

463 

464 with get_user_db_session(username) as session: 

465 # Get all benchmark runs (completed, failed, cancelled, or in-progress) 

466 runs = ( 

467 session.query(BenchmarkRun) 

468 .order_by(BenchmarkRun.created_at.desc()) 

469 .limit(50) 

470 .all() 

471 ) 

472 

473 # Format runs for display 

474 formatted_runs = [] 

475 for run in runs: 

476 # Calculate average processing time from results 

477 avg_processing_time = None 

478 avg_search_results = None 

479 try: 

480 from sqlalchemy import func 

481 

482 from ...database.models.benchmark import BenchmarkResult 

483 

484 avg_result = ( 

485 session.query(func.avg(BenchmarkResult.processing_time)) 

486 .filter( 

487 BenchmarkResult.benchmark_run_id == run.id, 

488 BenchmarkResult.processing_time.isnot(None), 

489 BenchmarkResult.processing_time > 0, 

490 ) 

491 .scalar() 

492 ) 

493 

494 if avg_result: 

495 avg_processing_time = float(avg_result) 

496 except Exception: 

497 logger.warning( 

498 f"Error calculating avg processing time for run {run.id}" 

499 ) 

500 

501 # Calculate average search results and total search requests from metrics 

502 total_search_requests = None 

503 try: 

504 from ...database.models import SearchCall 

505 

506 # Get all results for this run to find research_ids 

507 results = ( 

508 session.query(BenchmarkResult) 

509 .filter(BenchmarkResult.benchmark_run_id == run.id) 

510 .all() 

511 ) 

512 

513 research_ids = [ 

514 r.research_id for r in results if r.research_id 

515 ] 

516 

517 if research_ids: 

518 # SearchCall is in the same per-user DB, query directly 

519 search_calls = ( 

520 session.query(SearchCall) 

521 .filter(SearchCall.research_id.in_(research_ids)) 

522 .all() 

523 ) 

524 

525 # Group by research_id and calculate metrics per research session 

526 research_results = {} 

527 research_requests = {} 

528 

529 for call in search_calls: 

530 if call.research_id: 530 ↛ 529line 530 didn't jump to line 529 because the condition on line 530 was always true

531 if call.research_id not in research_results: 531 ↛ 534line 531 didn't jump to line 534 because the condition on line 531 was always true

532 research_results[call.research_id] = 0 

533 research_requests[call.research_id] = 0 

534 research_results[call.research_id] += ( 

535 call.results_count or 0 

536 ) 

537 research_requests[call.research_id] += 1 

538 

539 # Calculate averages across research sessions 

540 if research_results: 540 ↛ 556line 540 didn't jump to line 556 because the condition on line 540 was always true

541 total_results = sum(research_results.values()) 

542 avg_search_results = total_results / len( 

543 research_results 

544 ) 

545 

546 total_requests = sum(research_requests.values()) 

547 total_search_requests = total_requests / len( 

548 research_requests 

549 ) 

550 

551 except Exception: 

552 logger.warning( 

553 f"Error calculating search metrics for run {run.id}" 

554 ) 

555 

556 formatted_runs.append( 

557 { 

558 "id": run.id, 

559 "run_name": run.run_name or f"Benchmark #{run.id}", 

560 "created_at": run.created_at.isoformat(), 

561 # `start_time` is the wall-clock instant work began 

562 # (status flipped to IN_PROGRESS); used by the YAML 

563 # download for `date_tested` instead of the (broken) 

564 # download-time stamp. NULL on rows that never started. 

565 "start_time": run.start_time.isoformat() 

566 if run.start_time 

567 else None, 

568 # `ldr_version` is captured by start_benchmark; NULL 

569 # on rows created before migration 0014. The YAML 

570 # download substitutes "unknown (pre-0014 run)". 

571 # `settings_snapshot` is intentionally NOT included 

572 # here — it's ~184KB per run and would balloon the 

573 # history list response. Loaded on-demand from 

574 # /api/results/<id>/export when YAML is downloaded. 

575 "ldr_version": run.ldr_version, 

576 "total_examples": run.total_examples, 

577 "completed_examples": run.completed_examples, 

578 "overall_accuracy": run.overall_accuracy, 

579 "status": run.status.value, 

580 "search_config": run.search_config, 

581 "evaluation_config": run.evaluation_config, 

582 "datasets_config": run.datasets_config, 

583 "avg_processing_time": avg_processing_time, 

584 "avg_search_results": avg_search_results, 

585 "total_search_requests": total_search_requests, 

586 } 

587 ) 

588 

589 return {"success": True, "runs": formatted_runs} 

590 

591 except Exception: 

592 logger.exception("Error getting benchmark history") 

593 return JSONResponse( 

594 {"success": False, "error": "An internal error has occurred."}, 

595 status_code=500, 

596 ) 

597 

598 

599@router.get("/api/results/{benchmark_run_id}") 

600@limiter.exempt 

601def get_benchmark_results( 

602 request: Request, 

603 benchmark_run_id: int, 

604 username: Annotated[str, Depends(require_auth)], 

605): 

606 """Get detailed results for a benchmark run.""" 

607 try: 

608 from ...database.models.benchmark import BenchmarkResult 

609 from ...database.session_context import get_user_db_session 

610 

611 logger.info(f"Getting results for benchmark {benchmark_run_id}") 

612 

613 # First sync any pending results from active runs 

614 benchmark_service.sync_pending_results(benchmark_run_id, username) 

615 persistence_error = benchmark_service.get_result_persistence_error( 

616 benchmark_run_id, username 

617 ) 

618 with get_user_db_session(username) as session: 

619 # Get recent results (limit to last 10) 

620 try: 

621 limit = int(request.query_params.get("limit", 10)) 

622 except (TypeError, ValueError): 

623 limit = 10 

624 limit = max(1, min(limit, 1000)) 

625 

626 results = ( 

627 session.query(BenchmarkResult) 

628 .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id) 

629 # Results are returned unfiltered, including rows whose 

630 # evaluation is still pending — same as main's equivalent 

631 # route (benchmarks/web_api/benchmark_routes.py), which also 

632 # has no is_correct filter here. The is_correct.isnot(None) 

633 # filter belongs to the aggregate/stats queries in 

634 # benchmark_service.py, not to this listing. 

635 .order_by(BenchmarkResult.id.desc()) # Most recent first 

636 .limit(limit) 

637 .all() 

638 ) 

639 

640 logger.info(f"Found {len(results)} results") 

641 

642 # Build a map of research_id to total search results 

643 search_results_by_research_id = {} 

644 try: 

645 from ...database.models import SearchCall 

646 

647 # Get all unique research_ids from our results 

648 research_ids = [r.research_id for r in results if r.research_id] 

649 

650 if research_ids: 

651 # SearchCall is in the same per-user DB, query directly 

652 all_search_calls = ( 

653 session.query(SearchCall) 

654 .filter(SearchCall.research_id.in_(research_ids)) 

655 .all() 

656 ) 

657 

658 # Group search results by research_id 

659 for call in all_search_calls: 

660 if call.research_id: 660 ↛ 659line 660 didn't jump to line 659 because the condition on line 660 was always true

661 if ( 661 ↛ 668line 661 didn't jump to line 668 because the condition on line 661 was always true

662 call.research_id 

663 not in search_results_by_research_id 

664 ): 

665 search_results_by_research_id[ 

666 call.research_id 

667 ] = 0 

668 search_results_by_research_id[call.research_id] += ( 

669 call.results_count or 0 

670 ) 

671 

672 logger.info( 

673 f"Found search metrics for {len(search_results_by_research_id)} research IDs from {len(all_search_calls)} total search calls" 

674 ) 

675 logger.debug( 

676 f"Research IDs from results: {research_ids[:5] if len(research_ids) > 5 else research_ids}" 

677 ) 

678 logger.debug( 

679 f"Search results by research_id: {dict(list(search_results_by_research_id.items())[:5])}" 

680 ) 

681 except Exception: 

682 logger.exception( 

683 f"Error getting search metrics for benchmark {benchmark_run_id}" 

684 ) 

685 

686 # Format results for UI display 

687 formatted_results = [] 

688 for result in results: 

689 # Get search result count using research_id 

690 search_result_count = 0 

691 

692 try: 

693 if ( 

694 result.research_id 

695 and result.research_id in search_results_by_research_id 

696 ): 

697 search_result_count = search_results_by_research_id[ 

698 result.research_id 

699 ] 

700 logger.debug( 

701 f"Found {search_result_count} search results for research_id {result.research_id}" 

702 ) 

703 

704 except Exception: 

705 logger.exception( 

706 f"Error getting search results for result {result.example_id}" 

707 ) 

708 

709 formatted_results.append( 

710 { 

711 "example_id": result.example_id, 

712 "dataset_type": result.dataset_type.value, 

713 "question": result.question, 

714 "correct_answer": result.correct_answer, 

715 "model_answer": result.extracted_answer, 

716 "full_response": result.response, 

717 "is_correct": result.is_correct, 

718 "confidence": result.confidence, 

719 "grader_response": result.grader_response, 

720 "processing_time": result.processing_time, 

721 "search_result_count": search_result_count, 

722 "sources": result.sources, 

723 "completed_at": result.completed_at.isoformat() 

724 if result.completed_at 

725 else None, 

726 } 

727 ) 

728 

729 payload = {"success": True, "results": formatted_results} 

730 if persistence_error: 

731 payload["persistence_error"] = persistence_error 

732 return payload 

733 

734 except Exception: 

735 logger.exception("Error getting benchmark results") 

736 return JSONResponse( 

737 {"success": False, "error": "An internal error has occurred."}, 

738 status_code=500, 

739 ) 

740 

741 

742@router.get("/api/results/{benchmark_run_id}/export") 

743def export_benchmark_results( 

744 request: Request, 

745 benchmark_run_id: int, 

746 username: Annotated[str, Depends(require_auth)], 

747): 

748 """Get lightweight results for YAML export plus run-level provenance. 

749 

750 Returns a `metadata` block alongside the per-result rows so the YAML 

751 download path can stamp the version/settings that ran the benchmark 

752 instead of falling back to download-time values. Pre-0014 rows return 

753 NULL for the new metadata fields. The `success: True` envelope is 

754 preserved so the existing JS contract (5 callsites in 

755 benchmark_results.html) keeps working. 

756 """ 

757 try: 

758 from sqlalchemy.orm import load_only 

759 

760 from ...database.models.benchmark import ( 

761 BenchmarkResult, 

762 BenchmarkRun, 

763 ) 

764 from ...database.session_context import get_user_db_session 

765 

766 # The ~184KB settings snapshot is only loaded/returned when the 

767 # client opts in (Export -> "Include settings snapshot"); the 

768 # default summary download skips it to avoid transferring a blob it 

769 # would discard. 

770 include_settings = request.query_params.get( 

771 "include_settings", "" 

772 ).lower() in ( 

773 "1", 

774 "true", 

775 ) 

776 logger.info( 

777 "Exporting benchmark results for run {} by user {}", 

778 benchmark_run_id, 

779 username, 

780 ) 

781 with get_user_db_session(username) as session: 

782 # Fetch only the provenance columns we need from BenchmarkRun. 

783 # `load_only` keeps us off the heavy JSON config blobs (and the 

784 # settings_snapshot blob unless it was requested). 

785 run_columns = [ 

786 BenchmarkRun.ldr_version, 

787 BenchmarkRun.start_time, 

788 BenchmarkRun.created_at, 

789 ] 

790 if include_settings: 

791 run_columns.append(BenchmarkRun.settings_snapshot) 

792 run = ( 

793 session.query(BenchmarkRun) 

794 .options(load_only(*run_columns)) 

795 .filter(BenchmarkRun.id == benchmark_run_id) 

796 .one_or_none() 

797 ) 

798 

799 results = ( 

800 session.query(BenchmarkResult) 

801 .options( 

802 load_only( 

803 BenchmarkResult.example_id, 

804 BenchmarkResult.dataset_type, 

805 BenchmarkResult.question, 

806 BenchmarkResult.correct_answer, 

807 BenchmarkResult.extracted_answer, 

808 BenchmarkResult.is_correct, 

809 BenchmarkResult.confidence, 

810 BenchmarkResult.processing_time, 

811 BenchmarkResult.completed_at, 

812 ) 

813 ) 

814 .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id) 

815 .order_by(BenchmarkResult.id.asc()) 

816 .all() 

817 ) 

818 

819 formatted = [] 

820 for r in results: 

821 formatted.append( 

822 { 

823 "example_id": r.example_id, 

824 "dataset_type": r.dataset_type.value, 

825 "question": r.question, 

826 "correct_answer": r.correct_answer, 

827 "model_answer": r.extracted_answer, 

828 "is_correct": r.is_correct, 

829 "confidence": r.confidence, 

830 "processing_time": r.processing_time, 

831 "completed_at": r.completed_at.isoformat() 

832 if r.completed_at 

833 else None, 

834 } 

835 ) 

836 

837 # Build metadata block. Each field is independently null-checked 

838 # so a pre-0014 row, or an in-flight run with no start_time, 

839 # serializes cleanly as JSON null. 

840 if run is not None: 

841 started_at = ( 

842 run.start_time.isoformat() 

843 if run.start_time 

844 else ( 

845 run.created_at.isoformat() if run.created_at else None 

846 ) 

847 ) 

848 metadata = { 

849 "ldr_version": run.ldr_version, 

850 "started_at": started_at, 

851 "settings_snapshot": ( 

852 run.settings_snapshot if include_settings else None 

853 ), 

854 } 

855 else: 

856 metadata = { 

857 "ldr_version": None, 

858 "started_at": None, 

859 "settings_snapshot": None, 

860 } 

861 

862 logger.info( 

863 "Exported {} results for benchmark run {}", 

864 len(formatted), 

865 benchmark_run_id, 

866 ) 

867 return { 

868 "success": True, 

869 "metadata": metadata, 

870 "results": formatted, 

871 } 

872 

873 except Exception: 

874 logger.exception("Error exporting benchmark results") 

875 return JSONResponse( 

876 {"success": False, "error": "An internal error has occurred."}, 

877 status_code=500, 

878 ) 

879 

880 

881@router.get("/api/configs") 

882def get_saved_configs( 

883 request: Request, username: Annotated[str, Depends(require_auth)] 

884): 

885 """Get list of saved benchmark configurations.""" 

886 try: 

887 # Returns built-in default configs (saved configs not yet in DB) 

888 default_configs = [ 

889 { 

890 "id": 1, 

891 "name": "Quick Test", 

892 "description": "Fast benchmark with minimal examples", 

893 "search_config": { 

894 "iterations": 3, 

895 "questions_per_iteration": 3, 

896 "search_tool": "searxng", 

897 "search_strategy": "focused_iteration", 

898 }, 

899 "datasets_config": { 

900 "simpleqa": {"count": 10}, 

901 "browsecomp": {"count": 5}, 

902 }, 

903 }, 

904 { 

905 "id": 2,