Coverage for src/local_deep_research/benchmarks/runners.py: 100%
122 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Benchmark runners for Local Deep Research.
4This module provides the main functions for running benchmarks using LDR.
5"""
7import json
8from loguru import logger
9import os
10from pathlib import Path
11import time
12from typing import Any, Callable, Dict, Optional
14from ..api import quick_summary
15from .datasets import DEFAULT_DATASET_URLS, load_dataset
16from .datasets.base import DatasetRegistry
17from .graders import extract_answer_from_response, grade_results
18from .metrics import calculate_metrics, generate_report
19from .templates import BROWSECOMP_QUERY_TEMPLATE
22def format_query(question: str, dataset_type: str = "simpleqa") -> str:
23 """
24 Format query based on dataset type.
26 Args:
27 question: Original question
28 dataset_type: Type of dataset
30 Returns:
31 Formatted query for LDR
32 """
33 if dataset_type.lower() == "browsecomp":
34 # BrowseComp requires specific formatting
35 return BROWSECOMP_QUERY_TEMPLATE.format(question=question)
37 # Simple format for SimpleQA
38 return question
41def run_benchmark(
42 dataset_type: str,
43 dataset_path: Optional[str] = None,
44 num_examples: Optional[int] = None,
45 output_dir: str = "benchmark_results",
46 run_evaluation: bool = True,
47 evaluation_config: Optional[Dict[str, Any]] = None,
48 search_config: Optional[Dict[str, Any]] = None,
49 human_evaluation: bool = False,
50 progress_callback: Optional[Callable[[str, int, Dict], None]] = None,
51 seed: int = 42,
52 settings_snapshot: Optional[Dict[str, Any]] = None,
53) -> Dict[str, Any]:
54 """
55 Run a benchmark on the specified dataset.
57 Args:
58 dataset_type: Type of dataset ("simpleqa" or "browsecomp")
59 dataset_path: Optional custom dataset path
60 num_examples: Number of examples to use
61 output_dir: Directory to save results
62 run_evaluation: Whether to evaluate results
63 evaluation_config: Custom LLM config for evaluation
64 search_config: Custom search parameters
65 human_evaluation: Whether to use human evaluation
66 progress_callback: Optional callback for progress updates
67 seed: Random seed for reproducibility
69 Returns:
70 Dictionary with benchmark results and metrics
71 """
72 # Ensure output directory exists
73 os.makedirs(output_dir, exist_ok=True)
75 # Default search configuration
76 if not search_config:
77 search_config = {
78 "iterations": 3,
79 "questions_per_iteration": 3,
80 "search_tool": "searxng",
81 }
83 # Load dataset using the class-based approach
84 try:
85 # Create the dataset instance from registry
86 dataset_instance = DatasetRegistry.create_dataset(
87 dataset_id=dataset_type.lower(),
88 dataset_path=dataset_path,
89 num_examples=num_examples,
90 seed=seed,
91 )
92 # Load the examples
93 dataset = dataset_instance.load()
95 logger.info(
96 f"Loaded {len(dataset)} examples using dataset class {type(dataset_instance).__name__}"
97 )
98 except Exception:
99 # Fallback to legacy function if there's any issue
100 logger.warning(
101 "Error using dataset class. Falling back to legacy function."
102 )
103 # This is the local benchmark registry factory, not a Hugging Face download API.
104 dataset = load_dataset( # nosec B615
105 dataset_type=dataset_type,
106 dataset_path=dataset_path,
107 num_examples=num_examples,
108 seed=seed,
109 )
111 # Set up output files
112 timestamp = time.strftime("%Y%m%d_%H%M%S")
113 results_file = str(
114 Path(output_dir) / f"{dataset_type}_{timestamp}_results.jsonl"
115 )
116 evaluation_file = str(
117 Path(output_dir) / f"{dataset_type}_{timestamp}_evaluation.jsonl"
118 )
119 report_file = str(
120 Path(output_dir) / f"{dataset_type}_{timestamp}_report.md"
121 )
123 # Make sure output files don't exist
124 for file in [results_file, evaluation_file, report_file]:
125 file_path = Path(file)
126 if file_path.exists():
127 # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark output paths are selected by the invoking program; web benchmarks use the separate database runner
128 file_path.unlink()
130 # Progress tracking
131 total_examples = len(dataset)
133 if progress_callback:
134 progress_callback(
135 "Starting benchmark",
136 0,
137 {
138 "status": "started",
139 "dataset_type": dataset_type,
140 "total_examples": total_examples,
141 },
142 )
144 # Process each example
145 results = []
147 for i, example in enumerate(dataset):
148 # Extract question and answer in a way that uses the dataset class when available
149 if "dataset_instance" in locals() and isinstance(
150 dataset_instance,
151 DatasetRegistry.get_dataset_class(dataset_type.lower()),
152 ):
153 # Use the dataset class methods to extract question and answer
154 question = dataset_instance.get_question(example)
155 correct_answer = dataset_instance.get_answer(example)
156 logger.debug(
157 "Using dataset class methods to extract question and answer"
158 )
159 else:
160 # Fallback to the legacy approach
161 if dataset_type.lower() == "simpleqa":
162 question = example.get("problem", "")
163 correct_answer = example.get("answer", "")
164 else: # browsecomp
165 question = example.get("problem", "")
166 # For BrowseComp, the answer should be in "correct_answer" after decryption
167 correct_answer = example.get("correct_answer", "")
168 if not correct_answer and "answer" in example:
169 # Fallback to "answer" field if "correct_answer" is not available
170 correct_answer = example.get("answer", "")
172 # Update progress
173 if progress_callback:
174 progress_callback(
175 f"Processing example {i + 1}/{total_examples}",
176 int(i / total_examples * 50),
177 {
178 "status": "processing",
179 "current": i + 1,
180 "total": total_examples,
181 "question": (
182 question[:50] + "..."
183 if len(question) > 50
184 else question
185 ),
186 },
187 )
189 logger.info(f"Processing {i + 1}/{total_examples}: {question[:50]}...")
191 try:
192 # Format query based on dataset type
193 formatted_query = format_query(question, dataset_type)
195 # Time the search
196 start_time = time.time()
198 # Get response from LDR. Pass settings_snapshot through so
199 # the egress policy PEP fires on the benchmark's inner
200 # search-engine / LLM construction calls. Without it, the
201 # PEP's ``if settings_snapshot is not None`` guard skips and
202 # benchmarks silently bypass the user's saved scope.
203 search_result = quick_summary(
204 query=formatted_query,
205 iterations=search_config.get("iterations", 3),
206 questions_per_iteration=search_config.get(
207 "questions_per_iteration", 3
208 ),
209 search_tool=search_config.get("search_tool", "searxng"),
210 settings_snapshot=settings_snapshot,
211 )
213 end_time = time.time()
214 processing_time = end_time - start_time
216 # Extract response and search info
217 response = search_result.get("summary", "")
219 # Extract structured information
220 extracted = extract_answer_from_response(response, dataset_type)
222 # Format result
223 result = {
224 "id": example.get("id", f"example_{i}"),
225 "problem": question,
226 "correct_answer": correct_answer,
227 "response": response,
228 "extracted_answer": extracted["extracted_answer"],
229 "confidence": extracted["confidence"],
230 "processing_time": processing_time,
231 "sources": search_result.get("sources", []),
232 "search_config": search_config,
233 }
235 # Add to results list
236 results.append(result)
238 # Write result to file
239 # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark result file generated under its caller-selected output root
240 with open(results_file, "a", encoding="utf-8") as f:
241 f.write(json.dumps(result) + "\n")
243 # Update progress
244 if progress_callback:
245 progress_callback(
246 f"Completed example {i + 1}/{total_examples}",
247 int((i + 0.5) / total_examples * 50),
248 {
249 "status": "completed_example",
250 "current": i + 1,
251 "total": total_examples,
252 "result": result,
253 },
254 )
256 except Exception as e:
257 logger.exception(f"Error processing example {i + 1}")
259 # Create error result
260 error_result = {
261 "id": example.get("id", f"example_{i}"),
262 "problem": question,
263 "correct_answer": correct_answer,
264 "error": str(e),
265 "processing_time": (
266 time.time() - start_time if "start_time" in locals() else 0
267 ),
268 }
270 # Add to results list
271 results.append(error_result)
273 # Write error result to file
274 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Same generated benchmark result file used by the success path above
275 with open(results_file, "a", encoding="utf-8") as f:
276 f.write(json.dumps(error_result) + "\n")
278 # Update progress
279 if progress_callback:
280 progress_callback(
281 f"Error processing example {i + 1}/{total_examples}",
282 int((i + 0.5) / total_examples * 50),
283 {
284 "status": "error",
285 "current": i + 1,
286 "total": total_examples,
287 "error": str(e),
288 "result": error_result,
289 },
290 )
292 logger.info(f"Completed processing {total_examples} examples")
294 # Run evaluation if requested
295 if run_evaluation:
296 if progress_callback:
297 progress_callback(
298 "Starting evaluation",
299 50,
300 {"status": "evaluating", "results_file": results_file},
301 )
303 if human_evaluation:
304 from .graders import human_evaluation as evaluate
306 logger.info("Running human evaluation...")
307 evaluation_results = evaluate(
308 results_file=results_file,
309 output_file=evaluation_file,
310 interactive=True,
311 )
312 else:
313 logger.info("Running automated evaluation...")
314 try:
315 evaluation_results = grade_results(
316 results_file=results_file,
317 output_file=evaluation_file,
318 dataset_type=dataset_type,
319 evaluation_config=evaluation_config,
320 settings_snapshot=settings_snapshot,
321 progress_callback=lambda current, total, meta: (
322 progress_callback(
323 f"Evaluating {current + 1}/{total}",
324 50 + int((current + 0.5) / total * 40),
325 {**meta, "status": "evaluating"},
326 )
327 if progress_callback
328 else None
329 ),
330 )
331 except Exception as e:
332 logger.exception("Automated evaluation failed")
334 if progress_callback:
335 progress_callback(
336 "Automated evaluation failed. Falling back to human evaluation.",
337 60,
338 {"status": "evaluation_fallback", "error": str(e)},
339 )
341 # Ask if user wants to fall back to human evaluation
342 fallback_to_human = False
343 print("\nAutomated evaluation failed with error:", str(e))
344 response = input(
345 "Do you want to fall back to human evaluation? (y/n): "
346 )
347 fallback_to_human = response.strip().lower() == "y"
349 if fallback_to_human:
350 logger.info("Falling back to human evaluation...")
351 from .graders import human_evaluation as evaluate
353 evaluation_results = evaluate(
354 results_file=results_file,
355 output_file=evaluation_file,
356 interactive=True,
357 )
358 else:
359 from ..security.file_write_verifier import (
360 write_file_verified,
361 )
363 logger.info("Skipping evaluation due to error.")
364 # Create an empty evaluation file to prevent issues
365 write_file_verified(
366 evaluation_file,
367 "",
368 "benchmark.allow_file_output",
369 context="empty evaluation placeholder",
370 )
372 return {
373 "status": "evaluation_error",
374 "dataset_type": dataset_type,
375 "results_path": results_file,
376 "evaluation_error": str(e),
377 "total_examples": total_examples,
378 }
380 # Calculate metrics
381 if progress_callback:
382 progress_callback(
383 "Calculating metrics", 90, {"status": "calculating_metrics"}
384 )
386 metrics = calculate_metrics(evaluation_file)
388 # Generate report
389 if progress_callback:
390 progress_callback(
391 "Generating report", 95, {"status": "generating_report"}
392 )
394 dataset_name = dataset_type.capitalize()
395 report_path = generate_report(
396 metrics=metrics,
397 results_file=evaluation_file,
398 output_file=report_file,
399 dataset_name=dataset_name,
400 config_info={
401 "Dataset": dataset_path
402 or DEFAULT_DATASET_URLS.get(dataset_type, "Unknown"),
403 "Examples": total_examples,
404 "Iterations": search_config.get("iterations", 3),
405 "Questions per iteration": search_config.get(
406 "questions_per_iteration", 3
407 ),
408 "Search tool": search_config.get("search_tool", "searxng"),
409 "Evaluation method": "Human"
410 if human_evaluation
411 else "Automated",
412 },
413 )
415 # Mark as complete
416 if progress_callback:
417 progress_callback(
418 "Benchmark complete",
419 100,
420 {
421 "status": "complete",
422 "metrics": metrics,
423 "report_path": report_path,
424 },
425 )
427 return {
428 "status": "complete",
429 "dataset_type": dataset_type,
430 "results_path": results_file,
431 "evaluation_path": evaluation_file,
432 "report_path": report_path,
433 "metrics": metrics,
434 "total_examples": total_examples,
435 "accuracy": metrics.get("accuracy", 0),
436 }
438 # No evaluation, just return results
439 if progress_callback:
440 progress_callback(
441 "Benchmark complete (no evaluation)",
442 100,
443 {"status": "complete_no_eval", "results_path": results_file},
444 )
446 return {
447 "status": "complete_no_eval",
448 "dataset_type": dataset_type,
449 "results_path": results_file,
450 "total_examples": total_examples,
451 }
454def run_simpleqa_benchmark(num_examples: int = 100, **kwargs) -> Dict[str, Any]:
455 """
456 Run SimpleQA benchmark with default settings.
458 Args:
459 num_examples: Number of examples to process
460 **kwargs: Additional arguments to pass to run_benchmark
462 Returns:
463 Dictionary with benchmark results
464 """
465 return run_benchmark(
466 dataset_type="simpleqa", num_examples=num_examples, **kwargs
467 )
470def run_browsecomp_benchmark(
471 num_examples: int = 100, **kwargs
472) -> Dict[str, Any]:
473 """
474 Run BrowseComp benchmark with default settings.
476 Args:
477 num_examples: Number of examples to process
478 **kwargs: Additional arguments to pass to run_benchmark
480 Returns:
481 Dictionary with benchmark results
482 """
483 return run_benchmark(
484 dataset_type="browsecomp", num_examples=num_examples, **kwargs
485 )
488def run_xbench_deepsearch_benchmark(
489 num_examples: int = 100, **kwargs
490) -> Dict[str, Any]:
491 """
492 Run xbench-DeepSearch benchmark with default settings.
494 Args:
495 num_examples: Number of examples to process
496 **kwargs: Additional arguments to pass to run_benchmark
498 Returns:
499 Dictionary with benchmark results
500 """
501 return run_benchmark(
502 dataset_type="xbench_deepsearch", num_examples=num_examples, **kwargs
503 )