Coverage for src/local_deep_research/benchmarks/comparison/evaluator.py: 89%
314 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-19 23:35 +0000
1"""
2Configuration comparison for Local Deep Research.
4This module provides functions for comparing different parameter configurations
5and evaluating their performance across various metrics.
6"""
8import os
9from datetime import datetime, UTC
10from pathlib import Path
11from typing import Any, Dict, List, Optional
13import numpy as np
14from loguru import logger
16from local_deep_research.benchmarks.efficiency.resource_monitor import (
17 ResourceMonitor,
18)
19from local_deep_research.benchmarks.efficiency.speed_profiler import (
20 SpeedProfiler,
21)
22from local_deep_research.benchmarks.optimization.metrics import (
23 calculate_combined_score,
24 calculate_quality_metrics,
25 calculate_resource_metrics,
26 calculate_speed_metrics,
27)
28from local_deep_research.config.llm_config import get_llm
29from local_deep_research.config.search_config import get_search
30from local_deep_research.search_system import AdvancedSearchSystem
32# matplotlib is imported LAZILY via _ensure_plotting_loaded() inside the
33# visualization helpers below — NOT at module level. A module-level
34# `import matplotlib.pyplot` executes when this module is imported, and
35# because benchmarks/__init__.py pulls this module in, that import ran on
36# the server's import path. matplotlib's import is heavy and, under the
37# 2-core CI runner's GIL/CPU starvation, stretched to ~60s while holding
38# the import lock — freezing the whole werkzeug request pipeline (#4431).
39# These comparison visualizations only run in explicit benchmark
40# comparisons, never on a request path.
41#
42# Module-level placeholders so tests can @patch these names and so the
43# loader can fill them in place. None until first real visualization.
44plt = None
45Circle = None
46RegularPolygon = None
49def _ensure_plotting_loaded():
50 """Import matplotlib into module globals on first use (see #4431).
52 Early-returns if already loaded (or a test has patched plt) so it never
53 clobbers mocks.
54 """
55 global plt, Circle, RegularPolygon
56 if plt is not None:
57 return
58 import matplotlib.pyplot as plt
59 from matplotlib.patches import Circle, RegularPolygon
62def compare_configurations(
63 query: str,
64 configurations: List[Dict[str, Any]],
65 output_dir: str = "comparison_results",
66 model_name: Optional[str] = None,
67 provider: Optional[str] = None,
68 search_tool: Optional[str] = None,
69 repetitions: int = 1,
70 metric_weights: Optional[Dict[str, float]] = None,
71) -> Dict[str, Any]:
72 """
73 Compare multiple parameter configurations.
75 Args:
76 query: Research query to use for evaluation
77 configurations: List of parameter configurations to compare
78 output_dir: Directory to save comparison results
79 model_name: Name of the LLM model to use
80 provider: LLM provider
81 search_tool: Search engine to use
82 repetitions: Number of repetitions for each configuration
83 metric_weights: Dictionary of weights for each metric type
85 Returns:
86 Dictionary with comparison results
87 """
88 os.makedirs(output_dir, exist_ok=True)
90 # Default metric weights if not provided
91 if metric_weights is None:
92 metric_weights = {
93 "quality": 0.6,
94 "speed": 0.4,
95 "resource": 0.0, # Disabled by default
96 }
98 # Verify valid configurations
99 if not configurations:
100 logger.error("No configurations provided for comparison")
101 return {"error": "No configurations provided"}
103 # Results storage
104 results = []
106 # Process each configuration
107 for i, config in enumerate(configurations):
108 logger.info(
109 f"Evaluating configuration {i + 1}/{len(configurations)}: {config}"
110 )
112 # Name for this configuration
113 config_name = config.get("name", f"Configuration {i + 1}")
115 # Results for all repetitions of this configuration
116 config_results = []
118 # Run multiple repetitions
119 for rep in range(repetitions):
120 logger.info(
121 f"Starting repetition {rep + 1}/{repetitions} for {config_name}"
122 )
124 try:
125 # Run the configuration
126 result = _evaluate_single_configuration(
127 query=query,
128 config=config,
129 model_name=model_name,
130 provider=provider,
131 search_tool=search_tool,
132 )
134 config_results.append(result)
135 logger.info(f"Completed repetition {rep + 1} for {config_name}")
137 except Exception as e:
138 logger.exception(
139 f"Error in {config_name}, repetition {rep + 1}"
140 )
141 # Add error info but continue with other configurations
142 config_results.append({"error": str(e), "success": False})
144 # Calculate aggregate metrics across repetitions
145 if config_results: 145 ↛ 107line 145 didn't jump to line 107 because the condition on line 145 was always true
146 # Filter out failed runs
147 successful_runs = [
148 r for r in config_results if r.get("success", False)
149 ]
151 if successful_runs:
152 # Calculate average metrics
153 avg_metrics = _calculate_average_metrics(successful_runs)
155 # Calculate overall score
156 overall_score = calculate_combined_score(
157 metrics={
158 "quality": avg_metrics.get("quality_metrics", {}),
159 "speed": avg_metrics.get("speed_metrics", {}),
160 "resource": avg_metrics.get("resource_metrics", {}),
161 },
162 weights=metric_weights,
163 )
165 result_summary = {
166 "name": config_name,
167 "configuration": config,
168 "success": True,
169 "runs_completed": len(successful_runs),
170 "runs_failed": len(config_results) - len(successful_runs),
171 "avg_metrics": avg_metrics,
172 "overall_score": overall_score,
173 "individual_results": config_results,
174 }
175 else:
176 # All runs failed
177 result_summary = {
178 "name": config_name,
179 "configuration": config,
180 "success": False,
181 "runs_completed": 0,
182 "runs_failed": len(config_results),
183 "error": "All runs failed",
184 "individual_results": config_results,
185 }
187 results.append(result_summary)
189 # Sort results by overall score (if available)
190 sorted_results = sorted(
191 [r for r in results if r.get("success", False)],
192 key=lambda x: x.get("overall_score", 0),
193 reverse=True,
194 )
196 # Add failed configurations at the end
197 sorted_results.extend([r for r in results if not r.get("success", False)])
199 # Create comparison report
200 comparison_report = {
201 "query": query,
202 "configurations_tested": len(configurations),
203 "successful_configurations": len(
204 [r for r in results if r.get("success", False)]
205 ),
206 "failed_configurations": len(
207 [r for r in results if not r.get("success", False)]
208 ),
209 "repetitions": repetitions,
210 "metric_weights": metric_weights,
211 "timestamp": datetime.now(UTC).isoformat(),
212 "results": sorted_results,
213 }
215 # Save results to file
216 from ...security.file_write_verifier import write_json_verified
218 timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
219 result_file = str(Path(output_dir) / f"comparison_results_{timestamp}.json")
221 write_json_verified(
222 result_file,
223 comparison_report,
224 "benchmark.allow_file_output",
225 context="comparison results",
226 )
228 # Generate visualizations
229 _viz_dir_path = Path(output_dir) / "visualizations"
230 _viz_dir_path.mkdir(parents=True, exist_ok=True)
231 visualizations_dir = str(_viz_dir_path)
233 _create_comparison_visualizations(
234 comparison_report, output_dir=visualizations_dir, timestamp=timestamp
235 )
237 logger.info(f"Comparison completed. Results saved to {result_file}")
239 # Add report path to the result
240 comparison_report["report_path"] = result_file
242 return comparison_report
245def _evaluate_single_configuration(
246 query: str,
247 config: Dict[str, Any],
248 model_name: Optional[str] = None,
249 provider: Optional[str] = None,
250 search_tool: Optional[str] = None,
251) -> Dict[str, Any]:
252 """
253 Evaluate a single configuration.
255 Args:
256 query: Research query to evaluate
257 config: Configuration parameters
258 model_name: Name of the LLM model to use
259 provider: LLM provider
260 search_tool: Search engine to use
262 Returns:
263 Dictionary with evaluation results
264 """
265 # Extract configuration parameters
266 config_model_name = config.get("model_name", model_name)
267 config_provider = config.get("provider", provider)
268 config_search_tool = config.get("search_tool", search_tool)
269 config_iterations = config.get("iterations", 2)
270 config_questions_per_iteration = config.get("questions_per_iteration", 2)
271 config_search_strategy = config.get("search_strategy", "source-based")
273 # Initialize profiling tools
274 speed_profiler = SpeedProfiler()
275 resource_monitor = ResourceMonitor(sampling_interval=0.5)
277 # Start profiling
278 speed_profiler.start()
279 resource_monitor.start()
281 llm = None
282 search = None
283 system = None
284 try:
285 # Get LLM
286 with speed_profiler.timer("llm_initialization"):
287 llm = get_llm(
288 temperature=config.get("temperature", 0.7),
289 model_name=config_model_name,
290 provider=config_provider,
291 )
293 # Set up search engine if specified
294 with speed_profiler.timer("search_initialization"):
295 search = None
296 if config_search_tool: 296 ↛ 297line 296 didn't jump to line 297 because the condition on line 296 was never true
297 search = get_search(
298 config_search_tool,
299 llm_instance=llm,
300 )
302 # Create search system
303 system = AdvancedSearchSystem( # type: ignore[call-arg]
304 llm=llm,
305 search=search, # type: ignore[arg-type]
306 max_iterations=config_iterations,
307 questions_per_iteration=config_questions_per_iteration,
308 strategy_name=config_search_strategy,
309 )
311 # Run the analysis
312 with speed_profiler.timer("analysis"):
313 results = system.analyze_topic(query)
315 # Stop profiling
316 speed_profiler.stop()
317 resource_monitor.stop()
319 # Calculate metrics
320 _system_config = dict(config)
321 quality_metrics = calculate_quality_metrics(
322 system_config=_system_config
323 )
325 speed_metrics = calculate_speed_metrics(system_config=_system_config)
327 resource_metrics = calculate_resource_metrics(
328 system_config=_system_config
329 )
331 # Return comprehensive results
332 return {
333 "query": query,
334 "config": config,
335 "success": True,
336 "findings_count": len(results.get("findings", [])),
337 "knowledge_length": len(results.get("current_knowledge", "")),
338 "quality_metrics": quality_metrics,
339 "speed_metrics": speed_metrics,
340 "resource_metrics": resource_metrics,
341 "timing_details": speed_profiler.get_timings(),
342 "resource_details": resource_monitor.get_combined_stats(),
343 }
345 except Exception as e:
346 # Stop profiling on error
347 speed_profiler.stop()
348 resource_monitor.stop()
350 # Log the error
351 logger.exception("Error evaluating configuration")
353 # Return error information
354 return {
355 "query": query,
356 "config": config,
357 "success": False,
358 "error": str(e),
359 "timing_details": speed_profiler.get_timings(),
360 "resource_details": resource_monitor.get_combined_stats(),
361 }
362 finally:
363 from ...utilities.resource_utils import safe_close
365 safe_close(system, "evaluator system")
366 safe_close(search, "evaluator search engine")
367 safe_close(llm, "evaluator LLM")
370def _calculate_average_metrics(results: List[Dict[str, Any]]) -> Dict[str, Any]:
371 """
372 Calculate average metrics across multiple runs.
374 Args:
375 results: List of individual run results
377 Returns:
378 Dictionary with averaged metrics
379 """
380 # Check if there are any successful results
381 if not results:
382 return {}
384 # Initialize average metrics
385 avg_metrics: Dict[str, Any] = {
386 "quality_metrics": {},
387 "speed_metrics": {},
388 "resource_metrics": {},
389 }
391 # Quality metrics
392 quality_keys = set()
393 for result in results:
394 quality_metrics = result.get("quality_metrics", {})
395 quality_keys.update(quality_metrics.keys())
397 for key in quality_keys:
398 values = [r.get("quality_metrics", {}).get(key) for r in results]
399 values = [v for v in values if v is not None]
400 if values: 400 ↛ 397line 400 didn't jump to line 397 because the condition on line 400 was always true
401 avg_metrics["quality_metrics"][key] = sum(values) / len(values)
403 # Speed metrics
404 speed_keys = set()
405 for result in results:
406 speed_metrics = result.get("speed_metrics", {})
407 speed_keys.update(speed_metrics.keys())
409 for key in speed_keys:
410 values = [r.get("speed_metrics", {}).get(key) for r in results]
411 values = [v for v in values if v is not None]
412 if values: 412 ↛ 409line 412 didn't jump to line 409 because the condition on line 412 was always true
413 avg_metrics["speed_metrics"][key] = sum(values) / len(values)
415 # Resource metrics
416 resource_keys = set()
417 for result in results:
418 resource_metrics = result.get("resource_metrics", {})
419 resource_keys.update(resource_metrics.keys())
421 for key in resource_keys:
422 values = [r.get("resource_metrics", {}).get(key) for r in results]
423 values = [v for v in values if v is not None]
424 if values: 424 ↛ 421line 424 didn't jump to line 421 because the condition on line 424 was always true
425 avg_metrics["resource_metrics"][key] = sum(values) / len(values)
427 return avg_metrics
430def _create_comparison_visualizations(
431 comparison_report: Dict[str, Any], output_dir: str, timestamp: str
432):
433 """
434 Create visualizations for the comparison results.
436 Args:
437 comparison_report: Comparison report dictionary
438 output_dir: Directory to save visualizations
439 timestamp: Timestamp string for filenames
440 """
441 _ensure_plotting_loaded()
442 # Check if there are successful results
443 successful_results = [
444 r
445 for r in comparison_report.get("results", [])
446 if r.get("success", False)
447 ]
449 if not successful_results:
450 logger.warning("No successful configurations to visualize")
451 return
453 # Extract configuration names
454 config_names = [
455 r.get("name", f"Config {i + 1}")
456 for i, r in enumerate(successful_results)
457 ]
459 # 1. Overall score comparison
460 plt.figure(figsize=(12, 6))
461 scores = [r.get("overall_score", 0) for r in successful_results]
463 # Create horizontal bar chart
464 plt.barh(config_names, scores, color="skyblue")
465 plt.xlabel("Overall Score")
466 plt.ylabel("Configuration")
467 plt.title("Configuration Performance Comparison")
468 plt.grid(axis="x", linestyle="--", alpha=0.7)
469 plt.tight_layout()
470 plt.savefig(
471 str(Path(output_dir) / f"overall_score_comparison_{timestamp}.png")
472 )
473 plt.close()
475 # 2. Quality metrics comparison
476 quality_metrics = ["overall_quality", "source_count", "lexical_diversity"]
477 _create_metric_comparison_chart(
478 successful_results,
479 config_names,
480 quality_metrics,
481 "quality_metrics",
482 "Quality Metrics Comparison",
483 str(Path(output_dir) / f"quality_metrics_comparison_{timestamp}.png"),
484 )
486 # 3. Speed metrics comparison
487 speed_metrics = ["overall_speed", "total_duration", "duration_per_question"]
488 _create_metric_comparison_chart(
489 successful_results,
490 config_names,
491 speed_metrics,
492 "speed_metrics",
493 "Speed Metrics Comparison",
494 str(Path(output_dir) / f"speed_metrics_comparison_{timestamp}.png"),
495 )
497 # 4. Resource metrics comparison
498 resource_metrics = [
499 "overall_resource",
500 "process_memory_max_mb",
501 "system_cpu_avg",
502 ]
503 _create_metric_comparison_chart(
504 successful_results,
505 config_names,
506 resource_metrics,
507 "resource_metrics",
508 "Resource Usage Comparison",
509 str(Path(output_dir) / f"resource_metrics_comparison_{timestamp}.png"),
510 )
512 # 5. Spider chart for multi-dimensional comparison
513 _create_spider_chart(
514 successful_results,
515 config_names,
516 str(Path(output_dir) / f"spider_chart_comparison_{timestamp}.png"),
517 )
519 # 6. Pareto frontier chart for quality vs. speed
520 _create_pareto_chart(
521 successful_results,
522 str(Path(output_dir) / f"pareto_chart_comparison_{timestamp}.png"),
523 )
526def _create_metric_comparison_chart(
527 results: List[Dict[str, Any]],
528 config_names: List[str],
529 metric_keys: List[str],
530 metric_category: str,
531 title: str,
532 output_path: str,
533):
534 """
535 Create a chart comparing specific metrics across configurations.
537 Args:
538 results: List of configuration results
539 config_names: Names of configurations
540 metric_keys: Keys of metrics to compare
541 metric_category: Category of metrics (quality_metrics, speed_metrics, etc.)
542 title: Chart title
543 output_path: Path to save the chart
544 """
545 _ensure_plotting_loaded()
546 # Create figure with multiple subplots (one per metric)
547 fig, axes = plt.subplots(
548 len(metric_keys), 1, figsize=(12, 5 * len(metric_keys))
549 )
551 # Handle case with only one metric
552 if len(metric_keys) == 1:
553 axes = [axes]
555 for i, metric_key in enumerate(metric_keys):
556 ax = axes[i]
558 # Get metric values
559 metric_values = []
560 for result in results:
561 metrics = result.get("avg_metrics", {}).get(metric_category, {})
562 value = metrics.get(metric_key)
564 # Handle time values for better visualization
565 if "duration" in metric_key and value is not None:
566 # Convert to seconds if > 60 seconds, minutes if > 60 minutes
567 if value > 3600:
568 value = value / 3600 # Convert to hours
569 metric_key += " (hours)"
570 elif value > 60:
571 value = value / 60 # Convert to minutes
572 metric_key += " (minutes)"
573 else:
574 metric_key += " (seconds)"
576 metric_values.append(value if value is not None else 0)
578 # Create horizontal bar chart
579 bars = ax.barh(config_names, metric_values, color="lightblue")
580 ax.set_xlabel(metric_key.replace("_", " ").title())
581 ax.set_title(f"{metric_key.replace('_', ' ').title()}")
582 ax.grid(axis="x", linestyle="--", alpha=0.7)
584 # Add value labels to bars
585 for bar in bars:
586 width = bar.get_width()
587 label_x_pos = width * 1.01
588 ax.text(
589 label_x_pos,
590 bar.get_y() + bar.get_height() / 2,
591 f"{width:.2f}",
592 va="center",
593 )
595 plt.suptitle(title, fontsize=16)
596 plt.tight_layout()
597 plt.savefig(output_path)
598 plt.close()
601def _create_spider_chart(
602 results: List[Dict[str, Any]], config_names: List[str], output_path: str
603):
604 """
605 Create a spider chart comparing metrics across configurations.
607 Args:
608 results: List of configuration results
609 config_names: Names of configurations
610 output_path: Path to save the chart
611 """
612 _ensure_plotting_loaded()
613 # Try to import the radar chart module
614 try:
615 from matplotlib.path import Path
616 from matplotlib.projections import register_projection
617 from matplotlib.projections.polar import PolarAxes
618 from matplotlib.spines import Spine
620 def radar_factory(num_vars, frame="circle"):
621 """Create a radar chart with `num_vars` axes."""
622 # Calculate evenly-spaced axis angles
623 theta = np.linspace(0, 2 * np.pi, num_vars, endpoint=False)
625 class RadarAxes(PolarAxes):
626 name = "radar"
628 def __init__(self, *args, **kwargs):
629 super().__init__(*args, **kwargs)
630 self.set_theta_zero_location("N")
632 def fill(self, *args, closed=True, **kwargs):
633 return super().fill(closed=closed, *args, **kwargs)
635 def plot(self, *args, **kwargs):
636 return super().plot(*args, **kwargs)
638 def set_varlabels(self, labels):
639 self.set_thetagrids(np.degrees(theta), labels)
641 def _gen_axes_patch(self):
642 if frame == "circle":
643 return Circle((0.5, 0.5), 0.5)
644 if frame == "polygon":
645 return RegularPolygon(
646 (0.5, 0.5), num_vars, radius=0.5, edgecolor="k"
647 )
648 raise ValueError("Unknown value for 'frame': %s" % frame) # noqa: TRY301 — inside nested method definition, not caught by enclosing try
650 def _gen_axes_spines(self): # type: ignore[misc]
651 if frame == "circle":
652 return super()._gen_axes_spines()
653 if frame == "polygon":
654 spine_type = Spine.circular_spine
655 verts = unit_poly_verts(num_vars)
656 vertices = [(0.5, 0.5)] + verts
657 codes = (
658 [Path.MOVETO]
659 + [Path.LINETO] * num_vars
660 + [Path.CLOSEPOLY]
661 )
662 path = Path(vertices, codes)
663 spine = Spine(self, spine_type, path) # type: ignore[arg-type]
664 spine.set_transform(self.transAxes)
665 return {"polar": spine}
666 raise ValueError("Unknown value for 'frame': %s" % frame) # noqa: TRY301 — inside nested method definition, not caught by enclosing try
668 def unit_poly_verts(num_vars):
669 """Return vertices of polygon for radar chart."""
670 verts = []
671 for i in range(num_vars):
672 angle = theta[i]
673 verts.append(
674 (0.5 * (1 + np.cos(angle)), 0.5 * (1 + np.sin(angle)))
675 )
676 return verts
678 register_projection(RadarAxes)
679 return theta
681 # Select metrics for the spider chart
682 metrics = [
683 {"name": "Quality", "key": "quality_metrics.overall_quality"},
684 {"name": "Speed", "key": "speed_metrics.overall_speed"},
685 {
686 "name": "Sources",
687 "key": "quality_metrics.normalized_source_count",
688 },
689 {
690 "name": "Content",
691 "key": "quality_metrics.normalized_knowledge_length",
692 },
693 {
694 "name": "Memory",
695 "key": "resource_metrics.normalized_memory_usage",
696 "invert": True,
697 },
698 ]
700 # Extract metric values
701 spoke_labels = [m["name"] for m in metrics]
702 num_vars = len(spoke_labels)
703 theta = radar_factory(num_vars)
705 fig, ax = plt.subplots(
706 figsize=(10, 10), subplot_kw={"projection": "radar"}
707 )
709 # Color map for different configurations
710 colors = plt.cm.viridis(np.linspace(0, 1, len(results))) # type: ignore[attr-defined]
712 for i, result in enumerate(results):
713 values = []
714 for metric in metrics:
715 # Extract metric value using the key path (e.g., "quality_metrics.overall_quality")
716 key_parts = metric["key"].split(".")
717 value: Any = result.get("avg_metrics", {})
718 for part in key_parts:
719 value = value.get(part, 0) if isinstance(value, dict) else 0
721 # Invert if needed (for metrics where lower is better)
722 if metric.get("invert", False):
723 value = 1.0 - value
725 values.append(value)
727 # Plot this configuration
728 ax.plot(
729 theta,
730 values,
731 color=colors[i],
732 linewidth=2,
733 label=config_names[i],
734 )
735 ax.fill(theta, values, color=colors[i], alpha=0.25)
737 # Set chart properties
738 ax.set_varlabels(spoke_labels) # type: ignore[attr-defined]
739 plt.legend(loc="best", bbox_to_anchor=(0.5, 0.1))
740 plt.title("Multi-Dimensional Configuration Comparison", size=16, y=1.05)
741 plt.tight_layout()
743 # Save chart
744 plt.savefig(output_path)
745 plt.close()
747 except Exception as e:
748 logger.exception("Error creating spider chart")
749 # Create a text-based chart as fallback
750 plt.figure(figsize=(10, 6))
751 plt.text(
752 0.5,
753 0.5,
754 f"Spider chart could not be created: {e!s}",
755 horizontalalignment="center",
756 verticalalignment="center",
757 )
758 plt.axis("off")
759 plt.savefig(output_path)
760 plt.close()
763def _create_pareto_chart(results: List[Dict[str, Any]], output_path: str):
764 """
765 Create a Pareto frontier chart showing quality vs. speed tradeoff.
767 Args:
768 results: List of configuration results
769 output_path: Path to save the chart
770 """
771 _ensure_plotting_loaded()
772 # Extract quality and speed metrics
773 quality_scores = []
774 speed_scores = []
775 names = []
777 for result in results:
778 metrics = result.get("avg_metrics", {})
779 quality = metrics.get("quality_metrics", {}).get("overall_quality", 0)
781 # For speed, we use inverse of duration (so higher is better)
782 duration = metrics.get("speed_metrics", {}).get("total_duration", 1)
783 speed = 1.0 / max(duration, 0.001) # Avoid division by zero
785 quality_scores.append(quality)
786 speed_scores.append(speed)
787 names.append(result.get("name", "Configuration"))
789 # Create scatter plot
790 plt.figure(figsize=(10, 8))
791 plt.scatter(quality_scores, speed_scores, s=100, alpha=0.7)
793 # Add labels for each point
794 for i, name in enumerate(names):
795 plt.annotate(
796 name,
797 (quality_scores[i], speed_scores[i]),
798 xytext=(5, 5),
799 textcoords="offset points",
800 )
802 # Identify Pareto frontier
803 pareto_points = []
804 for i, (q, s) in enumerate(zip(quality_scores, speed_scores, strict=False)):
805 is_pareto = True
806 for q2, s2 in zip(quality_scores, speed_scores, strict=False):
807 if q2 > q and s2 > s: # Dominated
808 is_pareto = False
809 break
810 if is_pareto:
811 pareto_points.append(i)
813 # Highlight Pareto frontier
814 pareto_quality = [quality_scores[i] for i in pareto_points]
815 pareto_speed = [speed_scores[i] for i in pareto_points]
817 # Sort pareto points for line drawing
818 pareto_sorted = sorted(
819 zip(pareto_quality, pareto_speed, pareto_points, strict=False)
820 )
821 pareto_quality = [p[0] for p in pareto_sorted]
822 pareto_speed = [p[1] for p in pareto_sorted]
823 pareto_indices = [p[2] for p in pareto_sorted]
825 # Draw Pareto frontier line
826 plt.plot(pareto_quality, pareto_speed, "r--", linewidth=2)
828 # Highlight Pareto optimal points
829 plt.scatter(
830 [quality_scores[i] for i in pareto_indices],
831 [speed_scores[i] for i in pareto_indices],
832 s=150,
833 facecolors="none",
834 edgecolors="r",
835 linewidth=2,
836 )
838 # Add labels for Pareto optimal configurations
839 for i in pareto_indices:
840 plt.annotate(
841 names[i],
842 (quality_scores[i], speed_scores[i]),
843 xytext=(8, 8),
844 textcoords="offset points",
845 bbox={"boxstyle": "round,pad=0.5", "fc": "yellow", "alpha": 0.7},
846 )
848 # Set chart properties
849 plt.xlabel("Quality Score (higher is better)")
850 plt.ylabel("Speed Score (higher is better)")
851 plt.title("Quality vs. Speed Tradeoff (Pareto Frontier)", size=14)
852 plt.grid(True, linestyle="--", alpha=0.7)
854 # Add explanation
855 plt.figtext(
856 0.5,
857 0.01,
858 "Points on the red line are Pareto optimal configurations\n"
859 "(no other configuration is better in both quality and speed)",
860 ha="center",
861 fontsize=10,
862 bbox={"boxstyle": "round", "fc": "white", "alpha": 0.7},
863 )
865 plt.tight_layout()
866 plt.savefig(output_path)
867 plt.close()