Coverage for src/local_deep_research/benchmarks/comparison/evaluator.py: 89%
314 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Configuration comparison for Local Deep Research.
4This module provides functions for comparing different parameter configurations
5and evaluating their performance across various metrics.
6"""
8import os
9from datetime import datetime, UTC
10from pathlib import Path
11from typing import Any, Dict, List, Optional
13import numpy as np
14from loguru import logger
16from local_deep_research.benchmarks.efficiency.resource_monitor import (
17 ResourceMonitor,
18)
19from local_deep_research.benchmarks.efficiency.speed_profiler import (
20 SpeedProfiler,
21)
22from local_deep_research.benchmarks.optimization.metrics import (
23 calculate_combined_score,
24 calculate_quality_metrics,
25 calculate_resource_metrics,
26 calculate_speed_metrics,
27)
28from local_deep_research.config.llm_config import get_llm
29from local_deep_research.config.search_config import get_search
30from local_deep_research.search_system import AdvancedSearchSystem
32# matplotlib is imported LAZILY via _ensure_plotting_loaded() inside the
33# visualization helpers below — NOT at module level. A module-level
34# `import matplotlib.pyplot` executes when this module is imported, and
35# because benchmarks/__init__.py pulls this module in, that import ran on
36# the server's import path. matplotlib's import is heavy and, under the
37# 2-core CI runner's GIL/CPU starvation, stretched to ~60s while holding
38# the import lock — freezing the whole werkzeug request pipeline (#4431).
39# These comparison visualizations only run in explicit benchmark
40# comparisons, never on a request path.
41#
42# Module-level placeholders so tests can @patch these names and so the
43# loader can fill them in place. None until first real visualization.
44plt = None
45Circle = None
46RegularPolygon = None
49def _ensure_plotting_loaded():
50 """Import matplotlib into module globals on first use (see #4431).
52 Early-returns if already loaded (or a test has patched plt) so it never
53 clobbers mocks.
54 """
55 global plt, Circle, RegularPolygon
56 if plt is not None:
57 return
58 import matplotlib.pyplot as plt
59 from matplotlib.patches import Circle, RegularPolygon
62def compare_configurations(
63 query: str,
64 configurations: List[Dict[str, Any]],
65 output_dir: str = "comparison_results",
66 model_name: Optional[str] = None,
67 provider: Optional[str] = None,
68 search_tool: Optional[str] = None,
69 repetitions: int = 1,
70 metric_weights: Optional[Dict[str, float]] = None,
71) -> Dict[str, Any]:
72 """
73 Compare multiple parameter configurations.
75 Args:
76 query: Research query to use for evaluation
77 configurations: List of parameter configurations to compare
78 output_dir: Directory to save comparison results
79 model_name: Name of the LLM model to use
80 provider: LLM provider
81 search_tool: Search engine to use
82 repetitions: Number of repetitions for each configuration
83 metric_weights: Dictionary of weights for each metric type
85 Returns:
86 Dictionary with comparison results
87 """
88 os.makedirs(output_dir, exist_ok=True)
90 # Default metric weights if not provided
91 if metric_weights is None:
92 metric_weights = {
93 "quality": 0.6,
94 "speed": 0.4,
95 "resource": 0.0, # Disabled by default
96 }
98 # Verify valid configurations
99 if not configurations:
100 logger.error("No configurations provided for comparison")
101 return {"error": "No configurations provided"}
103 # Results storage
104 results = []
106 # Process each configuration
107 for i, config in enumerate(configurations):
108 logger.info(
109 f"Evaluating configuration {i + 1}/{len(configurations)}: {config}"
110 )
112 # Name for this configuration
113 config_name = config.get("name", f"Configuration {i + 1}")
115 # Results for all repetitions of this configuration
116 config_results = []
118 # Run multiple repetitions
119 for rep in range(repetitions):
120 logger.info(
121 f"Starting repetition {rep + 1}/{repetitions} for {config_name}"
122 )
124 try:
125 # Run the configuration
126 result = _evaluate_single_configuration(
127 query=query,
128 config=config,
129 model_name=model_name,
130 provider=provider,
131 search_tool=search_tool,
132 )
134 config_results.append(result)
135 logger.info(f"Completed repetition {rep + 1} for {config_name}")
137 except Exception as e:
138 logger.exception(
139 f"Error in {config_name}, repetition {rep + 1}"
140 )
141 # Add error info but continue with other configurations
142 config_results.append({"error": str(e), "success": False})
144 # Calculate aggregate metrics across repetitions
145 if config_results: 145 ↛ 107line 145 didn't jump to line 107 because the condition on line 145 was always true
146 # Filter out failed runs
147 successful_runs = [
148 r for r in config_results if r.get("success", False)
149 ]
151 if successful_runs:
152 # Calculate average metrics
153 avg_metrics = _calculate_average_metrics(successful_runs)
155 # Calculate overall score
156 overall_score = calculate_combined_score(
157 metrics={
158 "quality": avg_metrics.get("quality_metrics", {}),
159 "speed": avg_metrics.get("speed_metrics", {}),
160 "resource": avg_metrics.get("resource_metrics", {}),
161 },
162 weights=metric_weights,
163 )
165 result_summary = {
166 "name": config_name,
167 "configuration": config,
168 "success": True,
169 "runs_completed": len(successful_runs),
170 "runs_failed": len(config_results) - len(successful_runs),
171 "avg_metrics": avg_metrics,
172 "overall_score": overall_score,
173 "individual_results": config_results,
174 }
175 else:
176 # All runs failed
177 result_summary = {
178 "name": config_name,
179 "configuration": config,
180 "success": False,
181 "runs_completed": 0,
182 "runs_failed": len(config_results),
183 "error": "All runs failed",
184 "individual_results": config_results,
185 }
187 results.append(result_summary)
189 # Sort results by overall score (if available)
190 sorted_results = sorted(
191 [r for r in results if r.get("success", False)],
192 key=lambda x: x.get("overall_score", 0),
193 reverse=True,
194 )
196 # Add failed configurations at the end
197 sorted_results.extend([r for r in results if not r.get("success", False)])
199 # Create comparison report
200 comparison_report = {
201 "query": query,
202 "configurations_tested": len(configurations),
203 "successful_configurations": len(
204 [r for r in results if r.get("success", False)]
205 ),
206 "failed_configurations": len(
207 [r for r in results if not r.get("success", False)]
208 ),
209 "repetitions": repetitions,
210 "metric_weights": metric_weights,
211 "timestamp": datetime.now(UTC).isoformat(),
212 "results": sorted_results,
213 }
215 # Save results to file
216 from ...security.file_write_verifier import write_json_verified
218 timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
219 result_file = str(Path(output_dir) / f"comparison_results_{timestamp}.json")
221 write_json_verified(
222 result_file,
223 comparison_report,
224 "benchmark.allow_file_output",
225 context="comparison results",
226 )
228 # Generate visualizations
229 _viz_dir_path = Path(output_dir) / "visualizations"
230 # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python comparison output root with a fixed visualizations child; no web caller
231 _viz_dir_path.mkdir(parents=True, exist_ok=True)
232 visualizations_dir = str(_viz_dir_path)
234 _create_comparison_visualizations(
235 comparison_report, output_dir=visualizations_dir, timestamp=timestamp
236 )
238 logger.info(f"Comparison completed. Results saved to {result_file}")
240 # Add report path to the result
241 comparison_report["report_path"] = result_file
243 return comparison_report
246def _evaluate_single_configuration(
247 query: str,
248 config: Dict[str, Any],
249 model_name: Optional[str] = None,
250 provider: Optional[str] = None,
251 search_tool: Optional[str] = None,
252) -> Dict[str, Any]:
253 """
254 Evaluate a single configuration.
256 Args:
257 query: Research query to evaluate
258 config: Configuration parameters
259 model_name: Name of the LLM model to use
260 provider: LLM provider
261 search_tool: Search engine to use
263 Returns:
264 Dictionary with evaluation results
265 """
266 # Extract configuration parameters
267 config_model_name = config.get("model_name", model_name)
268 config_provider = config.get("provider", provider)
269 config_search_tool = config.get("search_tool", search_tool)
270 config_iterations = config.get("iterations", 2)
271 config_questions_per_iteration = config.get("questions_per_iteration", 2)
272 config_search_strategy = config.get("search_strategy", "source-based")
274 # Initialize profiling tools
275 speed_profiler = SpeedProfiler()
276 resource_monitor = ResourceMonitor(sampling_interval=0.5)
278 # Start profiling
279 speed_profiler.start()
280 resource_monitor.start()
282 llm = None
283 search = None
284 system = None
285 try:
286 # Get LLM
287 with speed_profiler.timer("llm_initialization"):
288 llm = get_llm(
289 temperature=config.get("temperature", 0.7),
290 model_name=config_model_name,
291 provider=config_provider,
292 )
294 # Set up search engine if specified
295 with speed_profiler.timer("search_initialization"):
296 search = None
297 if config_search_tool: 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true
298 search = get_search(
299 config_search_tool,
300 llm_instance=llm,
301 )
303 # Create search system
304 system = AdvancedSearchSystem( # type: ignore[call-arg]
305 llm=llm,
306 search=search, # type: ignore[arg-type]
307 max_iterations=config_iterations,
308 questions_per_iteration=config_questions_per_iteration,
309 strategy_name=config_search_strategy,
310 )
312 # Run the analysis
313 with speed_profiler.timer("analysis"):
314 results = system.analyze_topic(query)
316 # Stop profiling
317 speed_profiler.stop()
318 resource_monitor.stop()
320 # Calculate metrics
321 _system_config = dict(config)
322 quality_metrics = calculate_quality_metrics(
323 system_config=_system_config
324 )
326 speed_metrics = calculate_speed_metrics(system_config=_system_config)
328 resource_metrics = calculate_resource_metrics(
329 system_config=_system_config
330 )
332 # Return comprehensive results
333 return {
334 "query": query,
335 "config": config,
336 "success": True,
337 "findings_count": len(results.get("findings", [])),
338 "knowledge_length": len(results.get("current_knowledge", "")),
339 "quality_metrics": quality_metrics,
340 "speed_metrics": speed_metrics,
341 "resource_metrics": resource_metrics,
342 "timing_details": speed_profiler.get_timings(),
343 "resource_details": resource_monitor.get_combined_stats(),
344 }
346 except Exception as e:
347 # Stop profiling on error
348 speed_profiler.stop()
349 resource_monitor.stop()
351 # Log the error
352 logger.exception("Error evaluating configuration")
354 # Return error information
355 return {
356 "query": query,
357 "config": config,
358 "success": False,
359 "error": str(e),
360 "timing_details": speed_profiler.get_timings(),
361 "resource_details": resource_monitor.get_combined_stats(),
362 }
363 finally:
364 from ...utilities.resource_utils import safe_close
366 safe_close(system, "evaluator system")
367 safe_close(search, "evaluator search engine")
368 safe_close(llm, "evaluator LLM")
371def _calculate_average_metrics(results: List[Dict[str, Any]]) -> Dict[str, Any]:
372 """
373 Calculate average metrics across multiple runs.
375 Args:
376 results: List of individual run results
378 Returns:
379 Dictionary with averaged metrics
380 """
381 # Check if there are any successful results
382 if not results:
383 return {}
385 # Initialize average metrics
386 avg_metrics: Dict[str, Any] = {
387 "quality_metrics": {},
388 "speed_metrics": {},
389 "resource_metrics": {},
390 }
392 # Quality metrics
393 quality_keys = set()
394 for result in results:
395 quality_metrics = result.get("quality_metrics", {})
396 quality_keys.update(quality_metrics.keys())
398 for key in quality_keys:
399 values = [r.get("quality_metrics", {}).get(key) for r in results]
400 values = [v for v in values if v is not None]
401 if values: 401 ↛ 398line 401 didn't jump to line 398 because the condition on line 401 was always true
402 avg_metrics["quality_metrics"][key] = sum(values) / len(values)
404 # Speed metrics
405 speed_keys = set()
406 for result in results:
407 speed_metrics = result.get("speed_metrics", {})
408 speed_keys.update(speed_metrics.keys())
410 for key in speed_keys:
411 values = [r.get("speed_metrics", {}).get(key) for r in results]
412 values = [v for v in values if v is not None]
413 if values: 413 ↛ 410line 413 didn't jump to line 410 because the condition on line 413 was always true
414 avg_metrics["speed_metrics"][key] = sum(values) / len(values)
416 # Resource metrics
417 resource_keys = set()
418 for result in results:
419 resource_metrics = result.get("resource_metrics", {})
420 resource_keys.update(resource_metrics.keys())
422 for key in resource_keys:
423 values = [r.get("resource_metrics", {}).get(key) for r in results]
424 values = [v for v in values if v is not None]
425 if values: 425 ↛ 422line 425 didn't jump to line 422 because the condition on line 425 was always true
426 avg_metrics["resource_metrics"][key] = sum(values) / len(values)
428 return avg_metrics
431def _create_comparison_visualizations(
432 comparison_report: Dict[str, Any], output_dir: str, timestamp: str
433):
434 """
435 Create visualizations for the comparison results.
437 Args:
438 comparison_report: Comparison report dictionary
439 output_dir: Directory to save visualizations
440 timestamp: Timestamp string for filenames
441 """
442 _ensure_plotting_loaded()
443 # Check if there are successful results
444 successful_results = [
445 r
446 for r in comparison_report.get("results", [])
447 if r.get("success", False)
448 ]
450 if not successful_results:
451 logger.warning("No successful configurations to visualize")
452 return
454 # Extract configuration names
455 config_names = [
456 r.get("name", f"Config {i + 1}")
457 for i, r in enumerate(successful_results)
458 ]
460 # 1. Overall score comparison
461 plt.figure(figsize=(12, 6))
462 scores = [r.get("overall_score", 0) for r in successful_results]
464 # Create horizontal bar chart
465 plt.barh(config_names, scores, color="skyblue")
466 plt.xlabel("Overall Score")
467 plt.ylabel("Configuration")
468 plt.title("Configuration Performance Comparison")
469 plt.grid(axis="x", linestyle="--", alpha=0.7)
470 plt.tight_layout()
471 plt.savefig(
472 str(Path(output_dir) / f"overall_score_comparison_{timestamp}.png")
473 )
474 plt.close()
476 # 2. Quality metrics comparison
477 quality_metrics = ["overall_quality", "source_count", "lexical_diversity"]
478 _create_metric_comparison_chart(
479 successful_results,
480 config_names,
481 quality_metrics,
482 "quality_metrics",
483 "Quality Metrics Comparison",
484 str(Path(output_dir) / f"quality_metrics_comparison_{timestamp}.png"),
485 )
487 # 3. Speed metrics comparison
488 speed_metrics = ["overall_speed", "total_duration", "duration_per_question"]
489 _create_metric_comparison_chart(
490 successful_results,
491 config_names,
492 speed_metrics,
493 "speed_metrics",
494 "Speed Metrics Comparison",
495 str(Path(output_dir) / f"speed_metrics_comparison_{timestamp}.png"),
496 )
498 # 4. Resource metrics comparison
499 resource_metrics = [
500 "overall_resource",
501 "process_memory_max_mb",
502 "system_cpu_avg",
503 ]
504 _create_metric_comparison_chart(
505 successful_results,
506 config_names,
507 resource_metrics,
508 "resource_metrics",
509 "Resource Usage Comparison",
510 str(Path(output_dir) / f"resource_metrics_comparison_{timestamp}.png"),
511 )
513 # 5. Spider chart for multi-dimensional comparison
514 _create_spider_chart(
515 successful_results,
516 config_names,
517 str(Path(output_dir) / f"spider_chart_comparison_{timestamp}.png"),
518 )
520 # 6. Pareto frontier chart for quality vs. speed
521 _create_pareto_chart(
522 successful_results,
523 str(Path(output_dir) / f"pareto_chart_comparison_{timestamp}.png"),
524 )
527def _create_metric_comparison_chart(
528 results: List[Dict[str, Any]],
529 config_names: List[str],
530 metric_keys: List[str],
531 metric_category: str,
532 title: str,
533 output_path: str,
534):
535 """
536 Create a chart comparing specific metrics across configurations.
538 Args:
539 results: List of configuration results
540 config_names: Names of configurations
541 metric_keys: Keys of metrics to compare
542 metric_category: Category of metrics (quality_metrics, speed_metrics, etc.)
543 title: Chart title
544 output_path: Path to save the chart
545 """
546 _ensure_plotting_loaded()
547 # Create figure with multiple subplots (one per metric)
548 fig, axes = plt.subplots(
549 len(metric_keys), 1, figsize=(12, 5 * len(metric_keys))
550 )
552 # Handle case with only one metric
553 if len(metric_keys) == 1:
554 axes = [axes]
556 for i, metric_key in enumerate(metric_keys):
557 ax = axes[i]
559 # Get metric values
560 metric_values = []
561 for result in results:
562 metrics = result.get("avg_metrics", {}).get(metric_category, {})
563 value = metrics.get(metric_key)
565 # Handle time values for better visualization
566 if "duration" in metric_key and value is not None:
567 # Convert to seconds if > 60 seconds, minutes if > 60 minutes
568 if value > 3600:
569 value = value / 3600 # Convert to hours
570 metric_key += " (hours)"
571 elif value > 60:
572 value = value / 60 # Convert to minutes
573 metric_key += " (minutes)"
574 else:
575 metric_key += " (seconds)"
577 metric_values.append(value if value is not None else 0)
579 # Create horizontal bar chart
580 bars = ax.barh(config_names, metric_values, color="lightblue")
581 ax.set_xlabel(metric_key.replace("_", " ").title())
582 ax.set_title(f"{metric_key.replace('_', ' ').title()}")
583 ax.grid(axis="x", linestyle="--", alpha=0.7)
585 # Add value labels to bars
586 for bar in bars:
587 width = bar.get_width()
588 label_x_pos = width * 1.01
589 ax.text(
590 label_x_pos,
591 bar.get_y() + bar.get_height() / 2,
592 f"{width:.2f}",
593 va="center",
594 )
596 plt.suptitle(title, fontsize=16)
597 plt.tight_layout()
598 plt.savefig(output_path)
599 plt.close()
602def _create_spider_chart(
603 results: List[Dict[str, Any]], config_names: List[str], output_path: str
604):
605 """
606 Create a spider chart comparing metrics across configurations.
608 Args:
609 results: List of configuration results
610 config_names: Names of configurations
611 output_path: Path to save the chart
612 """
613 _ensure_plotting_loaded()
614 # Try to import the radar chart module
615 try:
616 from matplotlib.path import Path
617 from matplotlib.projections import register_projection
618 from matplotlib.projections.polar import PolarAxes
619 from matplotlib.spines import Spine
621 def radar_factory(num_vars, frame="circle"):
622 """Create a radar chart with `num_vars` axes."""
623 # Calculate evenly-spaced axis angles
624 theta = np.linspace(0, 2 * np.pi, num_vars, endpoint=False)
626 class RadarAxes(PolarAxes):
627 name = "radar"
629 def __init__(self, *args, **kwargs):
630 super().__init__(*args, **kwargs)
631 self.set_theta_zero_location("N")
633 def fill(self, *args, closed=True, **kwargs):
634 return super().fill(closed=closed, *args, **kwargs)
636 def plot(self, *args, **kwargs):
637 return super().plot(*args, **kwargs)
639 def set_varlabels(self, labels):
640 self.set_thetagrids(np.degrees(theta), labels)
642 def _gen_axes_patch(self):
643 if frame == "circle":
644 return Circle((0.5, 0.5), 0.5)
645 if frame == "polygon":
646 return RegularPolygon(
647 (0.5, 0.5), num_vars, radius=0.5, edgecolor="k"
648 )
649 raise ValueError("Unknown value for 'frame': %s" % frame) # noqa: TRY301 — inside nested method definition, not caught by enclosing try
651 def _gen_axes_spines(self): # type: ignore[misc]
652 if frame == "circle":
653 return super()._gen_axes_spines()
654 if frame == "polygon":
655 spine_type = Spine.circular_spine
656 verts = unit_poly_verts(num_vars)
657 vertices = [(0.5, 0.5)] + verts
658 codes = (
659 [Path.MOVETO]
660 + [Path.LINETO] * num_vars
661 + [Path.CLOSEPOLY]
662 )
663 path = Path(vertices, codes)
664 spine = Spine(self, spine_type, path) # type: ignore[arg-type]
665 spine.set_transform(self.transAxes)
666 return {"polar": spine}
667 raise ValueError("Unknown value for 'frame': %s" % frame) # noqa: TRY301 — inside nested method definition, not caught by enclosing try
669 def unit_poly_verts(num_vars):
670 """Return vertices of polygon for radar chart."""
671 verts = []
672 for i in range(num_vars):
673 angle = theta[i]
674 verts.append(
675 (0.5 * (1 + np.cos(angle)), 0.5 * (1 + np.sin(angle)))
676 )
677 return verts
679 register_projection(RadarAxes)
680 return theta
682 # Select metrics for the spider chart
683 metrics = [
684 {"name": "Quality", "key": "quality_metrics.overall_quality"},
685 {"name": "Speed", "key": "speed_metrics.overall_speed"},
686 {
687 "name": "Sources",
688 "key": "quality_metrics.normalized_source_count",
689 },
690 {
691 "name": "Content",
692 "key": "quality_metrics.normalized_knowledge_length",
693 },
694 {
695 "name": "Memory",
696 "key": "resource_metrics.normalized_memory_usage",
697 "invert": True,
698 },
699 ]
701 # Extract metric values
702 spoke_labels = [m["name"] for m in metrics]
703 num_vars = len(spoke_labels)
704 theta = radar_factory(num_vars)
706 fig, ax = plt.subplots(
707 figsize=(10, 10), subplot_kw={"projection": "radar"}
708 )
710 # Color map for different configurations
711 colors = plt.cm.viridis(np.linspace(0, 1, len(results))) # type: ignore[attr-defined]
713 for i, result in enumerate(results):
714 values = []
715 for metric in metrics:
716 # Extract metric value using the key path (e.g., "quality_metrics.overall_quality")
717 key_parts = metric["key"].split(".")
718 value: Any = result.get("avg_metrics", {})
719 for part in key_parts:
720 value = value.get(part, 0) if isinstance(value, dict) else 0
722 # Invert if needed (for metrics where lower is better)
723 if metric.get("invert", False):
724 value = 1.0 - value
726 values.append(value)
728 # Plot this configuration
729 ax.plot(
730 theta,
731 values,
732 color=colors[i],
733 linewidth=2,
734 label=config_names[i],
735 )
736 ax.fill(theta, values, color=colors[i], alpha=0.25)
738 # Set chart properties
739 ax.set_varlabels(spoke_labels) # type: ignore[attr-defined]
740 plt.legend(loc="best", bbox_to_anchor=(0.5, 0.1))
741 plt.title("Multi-Dimensional Configuration Comparison", size=16, y=1.05)
742 plt.tight_layout()
744 # Save chart
745 plt.savefig(output_path)
746 plt.close()
748 except Exception as e:
749 logger.exception("Error creating spider chart")
750 # Create a text-based chart as fallback
751 plt.figure(figsize=(10, 6))
752 plt.text(
753 0.5,
754 0.5,
755 f"Spider chart could not be created: {e!s}",
756 horizontalalignment="center",
757 verticalalignment="center",
758 )
759 plt.axis("off")
760 plt.savefig(output_path)
761 plt.close()
764def _create_pareto_chart(results: List[Dict[str, Any]], output_path: str):
765 """
766 Create a Pareto frontier chart showing quality vs. speed tradeoff.
768 Args:
769 results: List of configuration results
770 output_path: Path to save the chart
771 """
772 _ensure_plotting_loaded()
773 # Extract quality and speed metrics
774 quality_scores = []
775 speed_scores = []
776 names = []
778 for result in results:
779 metrics = result.get("avg_metrics", {})
780 quality = metrics.get("quality_metrics", {}).get("overall_quality", 0)
782 # For speed, we use inverse of duration (so higher is better)
783 duration = metrics.get("speed_metrics", {}).get("total_duration", 1)
784 speed = 1.0 / max(duration, 0.001) # Avoid division by zero
786 quality_scores.append(quality)
787 speed_scores.append(speed)
788 names.append(result.get("name", "Configuration"))
790 # Create scatter plot
791 plt.figure(figsize=(10, 8))
792 plt.scatter(quality_scores, speed_scores, s=100, alpha=0.7)
794 # Add labels for each point
795 for i, name in enumerate(names):
796 plt.annotate(
797 name,
798 (quality_scores[i], speed_scores[i]),
799 xytext=(5, 5),
800 textcoords="offset points",
801 )
803 # Identify Pareto frontier
804 pareto_points = []
805 for i, (q, s) in enumerate(zip(quality_scores, speed_scores, strict=False)):
806 is_pareto = True
807 for q2, s2 in zip(quality_scores, speed_scores, strict=False):
808 if q2 > q and s2 > s: # Dominated
809 is_pareto = False
810 break
811 if is_pareto:
812 pareto_points.append(i)
814 # Highlight Pareto frontier
815 pareto_quality = [quality_scores[i] for i in pareto_points]
816 pareto_speed = [speed_scores[i] for i in pareto_points]
818 # Sort pareto points for line drawing
819 pareto_sorted = sorted(
820 zip(pareto_quality, pareto_speed, pareto_points, strict=False)
821 )
822 pareto_quality = [p[0] for p in pareto_sorted]
823 pareto_speed = [p[1] for p in pareto_sorted]
824 pareto_indices = [p[2] for p in pareto_sorted]
826 # Draw Pareto frontier line
827 plt.plot(pareto_quality, pareto_speed, "r--", linewidth=2)
829 # Highlight Pareto optimal points
830 plt.scatter(
831 [quality_scores[i] for i in pareto_indices],
832 [speed_scores[i] for i in pareto_indices],
833 s=150,
834 facecolors="none",
835 edgecolors="r",
836 linewidth=2,
837 )
839 # Add labels for Pareto optimal configurations
840 for i in pareto_indices:
841 plt.annotate(
842 names[i],
843 (quality_scores[i], speed_scores[i]),
844 xytext=(8, 8),
845 textcoords="offset points",
846 bbox={"boxstyle": "round,pad=0.5", "fc": "yellow", "alpha": 0.7},
847 )
849 # Set chart properties
850 plt.xlabel("Quality Score (higher is better)")
851 plt.ylabel("Speed Score (higher is better)")
852 plt.title("Quality vs. Speed Tradeoff (Pareto Frontier)", size=14)
853 plt.grid(True, linestyle="--", alpha=0.7)
855 # Add explanation
856 plt.figtext(
857 0.5,
858 0.01,
859 "Points on the red line are Pareto optimal configurations\n"
860 "(no other configuration is better in both quality and speed)",
861 ha="center",
862 fontsize=10,
863 bbox={"boxstyle": "round", "fc": "white", "alpha": 0.7},
864 )
866 plt.tight_layout()
867 plt.savefig(output_path)
868 plt.close()