Coverage for src/local_deep_research/benchmarks/web_api/benchmark_routes.py: 96%

1"""Flask routes for benchmark web interface.""" 

2 

3import time 

4 

5from flask import Blueprint, jsonify, request 

6from loguru import logger 

7 

8from ...database.session_context import get_user_db_session 

9from ...security.decorators import require_json_body 

10from ...web.auth.decorators import login_required 

11from ...security.rate_limiter import limiter 

12from local_deep_research.settings import SettingsManager 

13from ...llm.providers.base import normalize_provider 

14from ...web.utils.templates import render_template_with_defaults 

15from .benchmark_service import benchmark_service 

16 

17# Create blueprint for benchmark routes 

18benchmark_bp = Blueprint("benchmark", __name__, url_prefix="/benchmark") 

19 

20# NOTE: Routes use flask_session["username"] (not .get()) intentionally. 

21# @login_required guarantees the key exists; direct access fails fast 

22# if the decorator is ever removed. 

23 

24 

25@benchmark_bp.route("/") 

26@login_required 

27def index(): 

28 """Benchmark dashboard page.""" 

29 from flask import session as flask_session 

30 

31 username = flask_session["username"] 

32 with get_user_db_session(username) as db_session: 

33 settings_manager = SettingsManager(db_session) 

34 

35 # Load evaluation settings from database 

36 eval_settings = { 

37 "evaluation_provider": settings_manager.get_setting( 

38 "benchmark.evaluation.provider", "openai_endpoint" 

39 ), 

40 "evaluation_model": settings_manager.get_setting( 

41 "benchmark.evaluation.model", "" 

42 ), 

43 "evaluation_endpoint_url": settings_manager.get_setting( 

44 "benchmark.evaluation.endpoint_url", "" 

45 ), 

46 "evaluation_temperature": settings_manager.get_setting( 

47 "benchmark.evaluation.temperature", 0 

48 ), 

49 } 

50 

51 return render_template_with_defaults( 

52 "pages/benchmark.html", eval_settings=eval_settings 

53 ) 

54 

55 

56@benchmark_bp.route("/results") 

57@login_required 

58def results(): 

59 """Benchmark results history page.""" 

60 return render_template_with_defaults("pages/benchmark_results.html") 

61 

62 

63@benchmark_bp.route("/api/start", methods=["POST"]) 

64@login_required 

65@require_json_body(error_message="No data provided") 

66def start_benchmark(): 

67 """Start a new benchmark run.""" 

68 try: 

69 data = request.get_json() 

70 

71 # Extract configuration 

72 run_name = data.get("run_name") 

73 

74 # Get search config from database instead of request 

75 from ...database.session_context import get_user_db_session 

76 from local_deep_research.settings import SettingsManager 

77 from flask import session as flask_session 

78 

79 username = flask_session["username"] 

80 session_id = flask_session.get("session_id") 

81 

82 # Try to get password from session store for background thread 

83 from ...database.session_passwords import session_password_store 

84 

85 user_password = None 

86 if session_id: 

87 user_password = session_password_store.get_session_password( 

88 username, session_id 

89 ) 

90 

91 search_config = {} 

92 evaluation_config = {} 

93 datasets_config = data.get("datasets_config", {}) 

94 

95 with get_user_db_session(username) as db_session: 

96 # Use the logged-in user's settings 

97 settings_manager = SettingsManager(db_session) 

98 

99 # Build search config from database settings 

100 search_config = { 

101 "iterations": int( 

102 settings_manager.get_setting("search.iterations", 8) 

103 ), 

104 "questions_per_iteration": int( 

105 settings_manager.get_setting( 

106 "search.questions_per_iteration", 5 

107 ) 

108 ), 

109 "search_tool": settings_manager.get_setting( 

110 "search.tool", "searxng" 

111 ), 

112 "search_strategy": settings_manager.get_setting( 

113 "search.search_strategy", "focused_iteration" 

114 ), 

115 "model_name": settings_manager.get_setting("llm.model"), 

116 "provider": settings_manager.get_setting("llm.provider"), 

117 "temperature": float( 

118 settings_manager.get_setting("llm.temperature", 0.7) 

119 ), 

120 "max_tokens": settings_manager.get_setting( 

121 "llm.max_tokens", 30000 

122 ), 

123 "context_window_unrestricted": settings_manager.get_setting( 

124 "llm.context_window_unrestricted", True 

125 ), 

126 "context_window_size": settings_manager.get_setting( 

127 "llm.context_window_size", 128000 

128 ), 

129 "local_context_window_size": settings_manager.get_setting( 

130 "llm.local_context_window_size", 8192 

131 ), 

132 } 

133 

134 # Add provider-specific settings 

135 provider = normalize_provider(search_config.get("provider")) 

136 if provider == "openai_endpoint": 

137 search_config["openai_endpoint_url"] = ( 

138 settings_manager.get_setting("llm.openai_endpoint.url") 

139 ) 

140 search_config["openai_endpoint_api_key"] = ( 

141 settings_manager.get_setting("llm.openai_endpoint.api_key") 

142 ) 

143 elif provider == "openai": 

144 search_config["openai_api_key"] = settings_manager.get_setting( 

145 "llm.openai.api_key" 

146 ) 

147 elif provider == "anthropic": 147 ↛ 153line 147 didn't jump to line 153 because the condition on line 147 was always true

148 search_config["anthropic_api_key"] = ( 

149 settings_manager.get_setting("llm.anthropic.api_key") 

150 ) 

151 

152 # Get evaluation config from database settings or request 

153 if "evaluation_config" in data: 

154 evaluation_config = data["evaluation_config"] 

155 else: 

156 # Read evaluation config from database settings 

157 evaluation_provider = normalize_provider( 

158 settings_manager.get_setting( 

159 "benchmark.evaluation.provider", "openai_endpoint" 

160 ) 

161 ) 

162 evaluation_model = settings_manager.get_setting( 

163 "benchmark.evaluation.model", "anthropic/claude-3.7-sonnet" 

164 ) 

165 evaluation_temperature = float( 

166 settings_manager.get_setting( 

167 "benchmark.evaluation.temperature", 0 

168 ) 

169 ) 

170 

171 evaluation_config = { 

172 "provider": evaluation_provider, 

173 "model_name": evaluation_model, 

174 "temperature": evaluation_temperature, 

175 } 

176 

177 # Add provider-specific settings for evaluation 

178 if evaluation_provider == "openai_endpoint": 

179 evaluation_config["openai_endpoint_url"] = ( 

180 settings_manager.get_setting( 

181 "benchmark.evaluation.endpoint_url", 

182 "https://openrouter.ai/api/v1", 

183 ) 

184 ) 

185 evaluation_config["openai_endpoint_api_key"] = ( 

186 settings_manager.get_setting( 

187 "llm.openai_endpoint.api_key" 

188 ) 

189 ) 

190 elif evaluation_provider == "openai": 

191 evaluation_config["openai_api_key"] = ( 

192 settings_manager.get_setting("llm.openai.api_key") 

193 ) 

194 elif evaluation_provider == "anthropic": 194 ↛ 200line 194 didn't jump to line 200

195 evaluation_config["anthropic_api_key"] = ( 

196 settings_manager.get_setting("llm.anthropic.api_key") 

197 ) 

198 

199 # Validate datasets config 

200 if not datasets_config or not any( 

201 config.get("count", 0) > 0 for config in datasets_config.values() 

202 ): 

203 return jsonify( 

204 { 

205 "error": "At least one dataset with count > 0 must be specified" 

206 } 

207 ), 400 

208 

209 # Create benchmark run 

210 benchmark_run_id = benchmark_service.create_benchmark_run( 

211 run_name=run_name, 

212 search_config=search_config, 

213 evaluation_config=evaluation_config, 

214 datasets_config=datasets_config, 

215 username=username, 

216 user_password=user_password, 

217 ) 

218 

219 # Start benchmark 

220 success = benchmark_service.start_benchmark( 

221 benchmark_run_id, username, user_password 

222 ) 

223 

224 if success: 

225 return jsonify( 

226 { 

227 "success": True, 

228 "benchmark_run_id": benchmark_run_id, 

229 "message": "Benchmark started successfully", 

230 } 

231 ) 

232 return jsonify( 

233 {"success": False, "error": "Failed to start benchmark"} 

234 ), 500 

235 

236 except Exception: 

237 logger.exception("Error starting benchmark") 

238 return jsonify( 

239 {"success": False, "error": "An internal error has occurred."} 

240 ), 500 

241 

242 

243@benchmark_bp.route("/api/running", methods=["GET"]) 

244@login_required 

245def get_running_benchmark(): 

246 """Check if there's a running benchmark and return its ID.""" 

247 try: 

248 from ...database.models.benchmark import BenchmarkRun, BenchmarkStatus 

249 from ...database.session_context import get_user_db_session 

250 from flask import session as flask_session 

251 

252 username = flask_session["username"] 

253 with get_user_db_session(username) as session: 

254 # Find any benchmark that's currently running 

255 running_benchmark = ( 

256 session.query(BenchmarkRun) 

257 .filter(BenchmarkRun.status == BenchmarkStatus.IN_PROGRESS) 

258 .order_by(BenchmarkRun.created_at.desc()) 

259 .first() 

260 ) 

261 

262 if running_benchmark: 

263 return jsonify( 

264 { 

265 "success": True, 

266 "benchmark_run_id": running_benchmark.id, 

267 "run_name": running_benchmark.run_name, 

268 "total_examples": running_benchmark.total_examples, 

269 "completed_examples": running_benchmark.completed_examples, 

270 } 

271 ) 

272 return jsonify( 

273 {"success": False, "message": "No running benchmark found"} 

274 ) 

275 

276 except Exception: 

277 logger.exception("Error checking for running benchmark") 

278 return jsonify( 

279 {"success": False, "error": "An internal error has occurred."} 

280 ), 500 

281 

282 

283@benchmark_bp.route("/api/status/<int:benchmark_run_id>", methods=["GET"]) 

284@limiter.exempt 

285@login_required 

286def get_benchmark_status(benchmark_run_id: int): 

287 """Get status of a benchmark run.""" 

288 try: 

289 from flask import session as flask_session 

290 

291 username = flask_session["username"] 

292 status = benchmark_service.get_benchmark_status( 

293 benchmark_run_id, username 

294 ) 

295 

296 if status: 

297 logger.info( 

298 f"Returning status for benchmark {benchmark_run_id}: " 

299 f"completed={status.get('completed_examples')}, " 

300 f"overall_acc={status.get('overall_accuracy')}, " 

301 f"avg_time={status.get('avg_time_per_example')}, " 

302 f"estimated_remaining={status.get('estimated_time_remaining')}" 

303 ) 

304 return jsonify({"success": True, "status": status}) 

305 return jsonify( 

306 {"success": False, "error": "Benchmark run not found"} 

307 ), 404 

308 

309 except Exception: 

310 logger.exception("Error getting benchmark status") 

311 return jsonify( 

312 {"success": False, "error": "An internal error has occurred."} 

313 ), 500 

314 

315 

316@benchmark_bp.route("/api/cancel/<int:benchmark_run_id>", methods=["POST"]) 

317@login_required 

318def cancel_benchmark(benchmark_run_id: int): 

319 """Cancel a running benchmark.""" 

320 try: 

321 from flask import session as flask_session 

322 

323 username = flask_session["username"] 

324 success = benchmark_service.cancel_benchmark(benchmark_run_id, username) 

325 

326 if success: 

327 return jsonify( 

328 {"success": True, "message": "Benchmark cancelled successfully"} 

329 ) 

330 return jsonify( 

331 {"success": False, "error": "Failed to cancel benchmark"} 

332 ), 500 

333 

334 except Exception: 

335 logger.exception("Error cancelling benchmark") 

336 return jsonify( 

337 {"success": False, "error": "An internal error has occurred."} 

338 ), 500 

339 

340 

341@benchmark_bp.route("/api/history", methods=["GET"]) 

342@login_required 

343def get_benchmark_history(): 

344 """Get list of recent benchmark runs.""" 

345 try: 

346 from ...database.models.benchmark import BenchmarkRun 

347 from ...database.session_context import get_user_db_session 

348 from flask import session as flask_session 

349 

350 username = flask_session["username"] 

351 with get_user_db_session(username) as session: 

352 # Get all benchmark runs (completed, failed, cancelled, or in-progress) 

353 runs = ( 

354 session.query(BenchmarkRun) 

355 .order_by(BenchmarkRun.created_at.desc()) 

356 .limit(50) 

357 .all() 

358 ) 

359 

360 # Format runs for display 

361 formatted_runs = [] 

362 for run in runs: 

363 # Calculate average processing time from results 

364 avg_processing_time = None 

365 avg_search_results = None 

366 try: 

367 from sqlalchemy import func 

368 

369 from ...database.models.benchmark import BenchmarkResult 

370 

371 avg_result = ( 

372 session.query(func.avg(BenchmarkResult.processing_time)) 

373 .filter( 

374 BenchmarkResult.benchmark_run_id == run.id, 

375 BenchmarkResult.processing_time.isnot(None), 

376 BenchmarkResult.processing_time > 0, 

377 ) 

378 .scalar() 

379 ) 

380 

381 if avg_result: 

382 avg_processing_time = float(avg_result) 

383 except Exception: 

384 logger.warning( 

385 f"Error calculating avg processing time for run {run.id}" 

386 ) 

387 

388 # Calculate average search results and total search requests from metrics 

389 total_search_requests = None 

390 try: 

391 from ...database.models import SearchCall 

392 

393 # Get all results for this run to find research_ids 

394 results = ( 

395 session.query(BenchmarkResult) 

396 .filter(BenchmarkResult.benchmark_run_id == run.id) 

397 .all() 

398 ) 

399 

400 research_ids = [ 

401 r.research_id for r in results if r.research_id 

402 ] 

403 

404 if research_ids: 

405 # SearchCall is in the same per-user DB, query directly 

406 search_calls = ( 

407 session.query(SearchCall) 

408 .filter(SearchCall.research_id.in_(research_ids)) 

409 .all() 

410 ) 

411 

412 # Group by research_id and calculate metrics per research session 

413 research_results = {} 

414 research_requests = {} 

415 

416 for call in search_calls: 

417 if call.research_id: 417 ↛ 416line 417 didn't jump to line 416 because the condition on line 417 was always true

418 if call.research_id not in research_results: 418 ↛ 421line 418 didn't jump to line 421 because the condition on line 418 was always true

419 research_results[call.research_id] = 0 

420 research_requests[call.research_id] = 0 

421 research_results[call.research_id] += ( 

422 call.results_count or 0 

423 ) 

424 research_requests[call.research_id] += 1 

425 

426 # Calculate averages across research sessions 

427 if research_results: 427 ↛ 443line 427 didn't jump to line 443 because the condition on line 427 was always true

428 total_results = sum(research_results.values()) 

429 avg_search_results = total_results / len( 

430 research_results 

431 ) 

432 

433 total_requests = sum(research_requests.values()) 

434 total_search_requests = total_requests / len( 

435 research_requests 

436 ) 

437 

438 except Exception: 

439 logger.warning( 

440 f"Error calculating search metrics for run {run.id}" 

441 ) 

442 

443 formatted_runs.append( 

444 { 

445 "id": run.id, 

446 "run_name": run.run_name or f"Benchmark #{run.id}", 

447 "created_at": run.created_at.isoformat(), 

448 # `start_time` is the wall-clock instant work began 

449 # (status flipped to IN_PROGRESS); used by the YAML 

450 # download for `date_tested` instead of the (broken) 

451 # download-time stamp. NULL on rows that never started. 

452 "start_time": run.start_time.isoformat() 

453 if run.start_time 

454 else None, 

455 # `ldr_version` is captured by start_benchmark; NULL 

456 # on rows created before migration 0014. The YAML 

457 # download substitutes "unknown (pre-0014 run)". 

458 # `settings_snapshot` is intentionally NOT included 

459 # here — it's ~184KB per run and would balloon the 

460 # history list response. Loaded on-demand from 

461 # /api/results/<id>/export when YAML is downloaded. 

462 "ldr_version": run.ldr_version, 

463 "total_examples": run.total_examples, 

464 "completed_examples": run.completed_examples, 

465 "overall_accuracy": run.overall_accuracy, 

466 "status": run.status.value, 

467 "search_config": run.search_config, 

468 "evaluation_config": run.evaluation_config, 

469 "datasets_config": run.datasets_config, 

470 "avg_processing_time": avg_processing_time, 

471 "avg_search_results": avg_search_results, 

472 "total_search_requests": total_search_requests, 

473 } 

474 ) 

475 

476 return jsonify({"success": True, "runs": formatted_runs}) 

477 

478 except Exception: 

479 logger.exception("Error getting benchmark history") 

480 return jsonify( 

481 {"success": False, "error": "An internal error has occurred."} 

482 ), 500 

483 

484 

485@benchmark_bp.route("/api/results/<int:benchmark_run_id>", methods=["GET"]) 

486@limiter.exempt 

487@login_required 

488def get_benchmark_results(benchmark_run_id: int): 

489 """Get detailed results for a benchmark run.""" 

490 try: 

491 from ...database.models.benchmark import BenchmarkResult 

492 from ...database.session_context import get_user_db_session 

493 from flask import session as flask_session 

494 

495 logger.info(f"Getting results for benchmark {benchmark_run_id}") 

496 username = flask_session["username"] 

497 

498 # First sync any pending results from active runs 

499 benchmark_service.sync_pending_results(benchmark_run_id, username) 

500 with get_user_db_session(username) as session: 

501 # Get recent results (limit to last 10) 

502 limit = int(request.args.get("limit", 10)) 

503 

504 results = ( 

505 session.query(BenchmarkResult) 

506 .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id) 

507 .order_by(BenchmarkResult.id.desc()) # Most recent first 

508 .limit(limit) 

509 .all() 

510 ) 

511 

512 logger.info(f"Found {len(results)} results") 

513 

514 # Build a map of research_id to total search results 

515 search_results_by_research_id = {} 

516 try: 

517 from ...database.models import SearchCall 

518 

519 # Get all unique research_ids from our results 

520 research_ids = [r.research_id for r in results if r.research_id] 

521 

522 if research_ids: 

523 # SearchCall is in the same per-user DB, query directly 

524 all_search_calls = ( 

525 session.query(SearchCall) 

526 .filter(SearchCall.research_id.in_(research_ids)) 

527 .all() 

528 ) 

529 

530 # Group search results by research_id 

531 for call in all_search_calls: 

532 if call.research_id: 532 ↛ 531line 532 didn't jump to line 531 because the condition on line 532 was always true

533 if ( 533 ↛ 540line 533 didn't jump to line 540 because the condition on line 533 was always true

534 call.research_id 

535 not in search_results_by_research_id 

536 ): 

537 search_results_by_research_id[ 

538 call.research_id 

539 ] = 0 

540 search_results_by_research_id[call.research_id] += ( 

541 call.results_count or 0 

542 ) 

543 

544 logger.info( 

545 f"Found search metrics for {len(search_results_by_research_id)} research IDs from {len(all_search_calls)} total search calls" 

546 ) 

547 logger.debug( 

548 f"Research IDs from results: {research_ids[:5] if len(research_ids) > 5 else research_ids}" 

549 ) 

550 logger.debug( 

551 f"Search results by research_id: {dict(list(search_results_by_research_id.items())[:5])}" 

552 ) 

553 except Exception: 

554 logger.exception( 

555 f"Error getting search metrics for benchmark {benchmark_run_id}" 

556 ) 

557 

558 # Format results for UI display 

559 formatted_results = [] 

560 for result in results: 

561 # Get search result count using research_id 

562 search_result_count = 0 

563 

564 try: 

565 if ( 

566 result.research_id 

567 and result.research_id in search_results_by_research_id 

568 ): 

569 search_result_count = search_results_by_research_id[ 

570 result.research_id 

571 ] 

572 logger.debug( 

573 f"Found {search_result_count} search results for research_id {result.research_id}" 

574 ) 

575 

576 except Exception: 

577 logger.exception( 

578 f"Error getting search results for result {result.example_id}" 

579 ) 

580 

581 formatted_results.append( 

582 { 

583 "example_id": result.example_id, 

584 "dataset_type": result.dataset_type.value, 

585 "question": result.question, 

586 "correct_answer": result.correct_answer, 

587 "model_answer": result.extracted_answer, 

588 "full_response": result.response, 

589 "is_correct": result.is_correct, 

590 "confidence": result.confidence, 

591 "grader_response": result.grader_response, 

592 "processing_time": result.processing_time, 

593 "search_result_count": search_result_count, 

594 "sources": result.sources, 

595 "completed_at": result.completed_at.isoformat() 

596 if result.completed_at 

597 else None, 

598 } 

599 ) 

600 

601 return jsonify({"success": True, "results": formatted_results}) 

602 

603 except Exception: 

604 logger.exception("Error getting benchmark results") 

605 return jsonify( 

606 {"success": False, "error": "An internal error has occurred."} 

607 ), 500 

608 

609 

610@benchmark_bp.route( 

611 "/api/results/<int:benchmark_run_id>/export", methods=["GET"] 

612) 

613@login_required 

614def export_benchmark_results(benchmark_run_id: int): 

615 """Get lightweight results for YAML export plus run-level provenance. 

616 

617 Returns a `metadata` block alongside the per-result rows so the YAML 

618 download path can stamp the version/settings that ran the benchmark 

619 instead of falling back to download-time values. Pre-0014 rows return 

620 NULL for the new metadata fields. The `success: True` envelope is 

621 preserved so the existing JS contract (5 callsites in 

622 benchmark_results.html) keeps working. 

623 """ 

624 try: 

625 from sqlalchemy.orm import load_only 

626 

627 from ...database.models.benchmark import ( 

628 BenchmarkResult, 

629 BenchmarkRun, 

630 ) 

631 from ...database.session_context import get_user_db_session 

632 from flask import request, session as flask_session 

633 

634 username = flask_session["username"] 

635 # The ~184KB settings snapshot is only loaded/returned when the 

636 # client opts in (Export -> "Include settings snapshot"); the 

637 # default summary download skips it to avoid transferring a blob it 

638 # would discard. 

639 include_settings = request.args.get("include_settings", "").lower() in ( 

640 "1", 

641 "true", 

642 ) 

643 logger.info( 

644 "Exporting benchmark results for run {} by user {}", 

645 benchmark_run_id, 

646 username, 

647 ) 

648 with get_user_db_session(username) as session: 

649 # Fetch only the provenance columns we need from BenchmarkRun. 

650 # `load_only` keeps us off the heavy JSON config blobs (and the 

651 # settings_snapshot blob unless it was requested). 

652 run_columns = [ 

653 BenchmarkRun.ldr_version, 

654 BenchmarkRun.start_time, 

655 BenchmarkRun.created_at, 

656 ] 

657 if include_settings: 

658 run_columns.append(BenchmarkRun.settings_snapshot) 

659 run = ( 

660 session.query(BenchmarkRun) 

661 .options(load_only(*run_columns)) 

662 .filter(BenchmarkRun.id == benchmark_run_id) 

663 .one_or_none() 

664 ) 

665 

666 results = ( 

667 session.query(BenchmarkResult) 

668 .options( 

669 load_only( 

670 BenchmarkResult.example_id, 

671 BenchmarkResult.dataset_type, 

672 BenchmarkResult.question, 

673 BenchmarkResult.correct_answer, 

674 BenchmarkResult.extracted_answer, 

675 BenchmarkResult.is_correct, 

676 BenchmarkResult.confidence, 

677 BenchmarkResult.processing_time, 

678 BenchmarkResult.completed_at, 

679 ) 

680 ) 

681 .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id) 

682 .order_by(BenchmarkResult.id.asc()) 

683 .all() 

684 ) 

685 

686 formatted = [] 

687 for r in results: 

688 formatted.append( 

689 { 

690 "example_id": r.example_id, 

691 "dataset_type": r.dataset_type.value, 

692 "question": r.question, 

693 "correct_answer": r.correct_answer, 

694 "model_answer": r.extracted_answer, 

695 "is_correct": r.is_correct, 

696 "confidence": r.confidence, 

697 "processing_time": r.processing_time, 

698 "completed_at": r.completed_at.isoformat() 

699 if r.completed_at 

700 else None, 

701 } 

702 ) 

703 

704 # Build metadata block. Each field is independently null-checked 

705 # so a pre-0014 row, or an in-flight run with no start_time, 

706 # serializes cleanly as JSON null. 

707 if run is not None: 

708 started_at = ( 

709 run.start_time.isoformat() 

710 if run.start_time 

711 else ( 

712 run.created_at.isoformat() if run.created_at else None 

713 ) 

714 ) 

715 metadata = { 

716 "ldr_version": run.ldr_version, 

717 "started_at": started_at, 

718 "settings_snapshot": ( 

719 run.settings_snapshot if include_settings else None 

720 ), 

721 } 

722 else: 

723 metadata = { 

724 "ldr_version": None, 

725 "started_at": None, 

726 "settings_snapshot": None, 

727 } 

728 

729 logger.info( 

730 "Exported {} results for benchmark run {}", 

731 len(formatted), 

732 benchmark_run_id, 

733 ) 

734 return jsonify( 

735 { 

736 "success": True, 

737 "metadata": metadata, 

738 "results": formatted, 

739 } 

740 ) 

741 

742 except Exception: 

743 logger.exception("Error exporting benchmark results") 

744 return jsonify( 

745 {"success": False, "error": "An internal error has occurred."} 

746 ), 500 

747 

748 

749@benchmark_bp.route("/api/configs", methods=["GET"]) 

750@login_required 

751def get_saved_configs(): 

752 """Get list of saved benchmark configurations.""" 

753 try: 

754 # TODO: Implement saved configs retrieval from database 

755 # For now return default configs 

756 default_configs = [ 

757 { 

758 "id": 1, 

759 "name": "Quick Test", 

760 "description": "Fast benchmark with minimal examples", 

761 "search_config": { 

762 "iterations": 3, 

763 "questions_per_iteration": 3, 

764 "search_tool": "searxng", 

765 "search_strategy": "focused_iteration", 

766 }, 

767 "datasets_config": { 

768 "simpleqa": {"count": 10}, 

769 "browsecomp": {"count": 5}, 

770 }, 

771 }, 

772 { 

773 "id": 2, 

774 "name": "Standard Evaluation", 

775 "description": "Comprehensive benchmark with standard settings", 

776 "search_config": { 

777 "iterations": 8, 

778 "questions_per_iteration": 5, 

779 "search_tool": "searxng", 

780 "search_strategy": "focused_iteration", 

781 }, 

782 "datasets_config": { 

783 "simpleqa": {"count": 50}, 

784 "browsecomp": {"count": 25}, 

785 }, 

786 }, 

787 ] 

788 

789 return jsonify({"success": True, "configs": default_configs}) 

790 

791 except Exception: 

792 logger.exception("Error getting saved configs") 

793 return jsonify( 

794 {"success": False, "error": "An internal error has occurred."} 

795 ), 500 

796 

797 

798@benchmark_bp.route("/api/start-simple", methods=["POST"]) 

799@login_required 

800@require_json_body() 

801def start_benchmark_simple(): 

802 """Start a benchmark using current database settings.""" 

803 try: 

804 data = request.get_json() 

805 datasets_config = data.get("datasets_config", {}) 

806 

807 # Validate datasets 

808 if not datasets_config or not any( 

809 config.get("count", 0) > 0 for config in datasets_config.values() 

810 ): 

811 return jsonify( 

812 { 

813 "error": "At least one dataset with count > 0 must be specified" 

814 } 

815 ), 400 

816 

817 # Get current settings from database 

818 from flask import session as flask_session 

819 

820 username = flask_session["username"] 

821 session_id = flask_session.get("session_id") 

822 

823 # Try to get password from session store for background thread 

824 from ...database.session_passwords import session_password_store 

825 

826 user_password = None 

827 if session_id: 

828 user_password = session_password_store.get_session_password( 

829 username, session_id 

830 ) 

831 

832 with get_user_db_session(username, user_password) as session: 

833 # For benchmarks, use a default test username 

834 settings_manager = SettingsManager(session) 

835 

836 # Build search config from database settings 

837 search_config = { 

838 "iterations": int( 

839 settings_manager.get_setting("search.iterations", 8) 

840 ), 

841 "questions_per_iteration": int( 

842 settings_manager.get_setting( 

843 "search.questions_per_iteration", 5 

844 ) 

845 ), 

846 "search_tool": settings_manager.get_setting( 

847 "search.tool", "searxng" 

848 ), 

849 "search_strategy": settings_manager.get_setting( 

850 "search.search_strategy", "focused_iteration" 

851 ), 

852 "model_name": settings_manager.get_setting("llm.model"), 

853 "provider": settings_manager.get_setting("llm.provider"), 

854 "temperature": float( 

855 settings_manager.get_setting("llm.temperature", 0.7) 

856 ), 

857 "max_tokens": settings_manager.get_setting( 

858 "llm.max_tokens", 30000 

859 ), 

860 "context_window_unrestricted": settings_manager.get_setting( 

861 "llm.context_window_unrestricted", True 

862 ), 

863 "context_window_size": settings_manager.get_setting( 

864 "llm.context_window_size", 128000 

865 ), 

866 "local_context_window_size": settings_manager.get_setting( 

867 "llm.local_context_window_size", 8192 

868 ), 

869 } 

870 

871 # Add provider-specific settings 

872 provider = normalize_provider(search_config.get("provider")) 

873 if provider == "openai_endpoint": 

874 search_config["openai_endpoint_url"] = ( 

875 settings_manager.get_setting("llm.openai_endpoint.url") 

876 ) 

877 search_config["openai_endpoint_api_key"] = ( 

878 settings_manager.get_setting("llm.openai_endpoint.api_key") 

879 ) 

880 elif provider == "openai": 

881 search_config["openai_api_key"] = settings_manager.get_setting( 

882 "llm.openai.api_key" 

883 ) 

884 elif provider == "anthropic": 884 ↛ 890line 884 didn't jump to line 890 because the condition on line 884 was always true

885 search_config["anthropic_api_key"] = ( 

886 settings_manager.get_setting("llm.anthropic.api_key") 

887 ) 

888 

889 # Read evaluation config from database settings 

890 evaluation_provider = normalize_provider( 

891 settings_manager.get_setting( 

892 "benchmark.evaluation.provider", "openai_endpoint" 

893 ) 

894 ) 

895 evaluation_model = settings_manager.get_setting( 

896 "benchmark.evaluation.model", "anthropic/claude-3.7-sonnet" 

897 ) 

898 evaluation_temperature = float( 

899 settings_manager.get_setting( 

900 "benchmark.evaluation.temperature", 0 

901 ) 

902 ) 

903 

904 evaluation_config = { 

905 "provider": evaluation_provider, 

906 "model_name": evaluation_model, 

907 "temperature": evaluation_temperature,