Coverage for src/local_deep_research/benchmarks/graders.py: 88%
245 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Evaluation and grading functionality.
4This module provides tools for evaluating model outputs against reference answers.
5"""
7import json
8from loguru import logger
9from pathlib import Path
10import re
11import time
12from typing import Any, Callable, Dict, List, Optional
14from langchain_core.messages.human import HumanMessage
16from ..config.llm_config import get_llm
17from ..llm.providers.base import normalize_provider
18from ..utilities.json_utils import get_llm_response_text
19from .templates import BROWSECOMP_GRADER_TEMPLATE, SIMPLEQA_GRADER_TEMPLATE
22# Default evaluation configuration using Claude 3.7 Sonnet via OpenRouter
23DEFAULT_EVALUATION_CONFIG = {
24 "model_name": "anthropic/claude-3.7-sonnet", # Correct model ID for OpenRouter
25 "provider": "openai_endpoint", # Use OpenRouter
26 "openai_endpoint_url": "https://openrouter.ai/api/v1", # OpenRouter URL
27 "temperature": 0, # Zero temp for consistent evaluation
28 # Note: max_tokens removed as it's not supported by LDR's get_llm()
29}
32def get_evaluation_llm(
33 custom_config: Optional[Dict[str, Any]] = None,
34 settings_snapshot: Optional[Dict[str, Any]] = None,
35):
36 """
37 Get an LLM for evaluation purposes using Claude 3.7 Sonnet via OpenRouter
38 by default, which can be overridden with custom settings.
40 Args:
41 custom_config: Optional custom configuration that overrides defaults
42 settings_snapshot: Optional settings snapshot for thread-safe access
44 Returns:
45 An LLM instance for evaluation
46 """
47 # Start with default config (Claude 3.7 Sonnet via OpenRouter)
48 config = DEFAULT_EVALUATION_CONFIG.copy()
50 # Override with any custom settings
51 if custom_config:
52 config.update(custom_config)
54 logger.info(
55 f"Getting evaluation LLM with provider={config['provider']}, model={config['model_name']}"
56 )
58 # Remove any parameters that LDR's get_llm doesn't support
59 # This ensures compatibility with LDR's implementation
60 ldr_supported_params = {
61 "model_name",
62 "temperature",
63 "provider",
64 "openai_endpoint_url",
65 "api_key",
66 }
68 filtered_config = {
69 k: v for k, v in config.items() if k in ldr_supported_params
70 }
72 # Check if we're using openai_endpoint but don't have an API key configured
73 if normalize_provider(filtered_config.get("provider")) == "openai_endpoint":
74 # Try to get API key from settings snapshot or environment
75 api_key = None
77 if settings_snapshot:
78 # Get from settings snapshot for thread safety
79 api_key_setting = settings_snapshot.get(
80 "llm.openai_endpoint.api_key"
81 )
82 if api_key_setting:
83 api_key = (
84 api_key_setting.get("value")
85 if isinstance(api_key_setting, dict)
86 else api_key_setting
87 )
88 else:
89 # No settings snapshot available
90 logger.warning(
91 "No settings snapshot provided for benchmark grader. "
92 "API key must be provided via settings_snapshot for thread safety."
93 )
95 if not api_key:
96 logger.warning(
97 "Using openai_endpoint provider but no API key found. "
98 "Set the llm.openai_endpoint.api_key setting in the database or "
99 "LDR_LLM_OPENAI_ENDPOINT_API_KEY environment variable."
100 )
101 # Try to fall back to LDR's config if API key not explicitly provided
102 # The get_llm function will handle this case
104 # Get the LLM using LDR's existing function. Thread settings_snapshot
105 # through — without it, get_llm's snapshot-less PEP only permits local
106 # default providers, so a cloud grader (e.g. openai) would be refused
107 # with PolicyDeniedError. The snapshot lets get_llm evaluate the policy
108 # and permit the configured grader when the user's scope allows it.
109 return get_llm(**filtered_config, settings_snapshot=settings_snapshot)
112def extract_answer_from_response(
113 response: str, dataset_type: str = "simpleqa"
114) -> Dict[str, str]:
115 """
116 Extract structured information from LDR's response.
118 Args:
119 response: Response from LDR
120 dataset_type: Type of dataset
122 Returns:
123 Dictionary with extracted answer and confidence
124 """
125 # Clean up citations — strip both ASCII "[N]" and lenticular "【N】"
126 # so a lenticular citation (some LLMs emit them) doesn't survive into
127 # the graded answer text and skew the match.
128 response = re.sub(r"[\[【]\d+[\]】]", "", response)
130 # Extract differently based on dataset type
131 if dataset_type.lower() == "browsecomp":
132 # Extract the final answer from structured response
133 answer_match = re.search(r"Exact Answer:\s*(.*?)(?:\n|$)", response)
134 exact_answer = answer_match.group(1).strip() if answer_match else "None"
136 # Extract confidence
137 confidence_match = re.search(r"Confidence:\s*(\d+)%", response)
138 confidence = confidence_match.group(1) if confidence_match else "100"
140 return {"extracted_answer": exact_answer, "confidence": confidence}
142 # For SimpleQA, return the whole response as the answer
143 return {
144 "extracted_answer": response,
145 "confidence": "100", # SimpleQA doesn't have confidence scores
146 }
149def _select_grader_template(dataset_type: str) -> str:
150 """Pick the grading template for a dataset type."""
151 return (
152 BROWSECOMP_GRADER_TEMPLATE
153 if dataset_type.lower() == "browsecomp"
154 else SIMPLEQA_GRADER_TEMPLATE
155 )
158def _build_grading_prompt(template: str, result_data: Dict[str, Any]) -> str:
159 """Format the grading prompt for one result."""
160 return template.format(
161 question=result_data.get("problem", ""),
162 correct_answer=result_data.get("correct_answer", ""),
163 response=result_data.get("response", ""),
164 )
167def _call_grader_llm(evaluation_llm, grading_prompt: str):
168 """Invoke synchronously and normalize supported text/content-block responses."""
169 if hasattr(evaluation_llm, "invoke") and callable(evaluation_llm.invoke):
170 prompt = (
171 [HumanMessage(content=grading_prompt)]
172 if hasattr(evaluation_llm, "chat_messages")
173 else grading_prompt
174 )
175 response = evaluation_llm.invoke(prompt)
176 else:
177 response = evaluation_llm(grading_prompt)
178 return get_llm_response_text(response)
181async def _acall_grader_llm(evaluation_llm, grading_prompt: str):
182 """Await the async interface, retaining the injected sync-only fallback."""
183 if hasattr(evaluation_llm, "ainvoke") and callable(evaluation_llm.ainvoke): 183 ↛ 191line 183 didn't jump to line 191 because the condition on line 183 was always true
184 prompt = (
185 [HumanMessage(content=grading_prompt)]
186 if hasattr(evaluation_llm, "chat_messages")
187 else grading_prompt
188 )
189 response = await evaluation_llm.ainvoke(prompt)
190 return get_llm_response_text(response)
191 return _call_grader_llm(evaluation_llm, grading_prompt)
194def _parse_grading_response(
195 grading_response: str, dataset_type: str
196) -> Dict[str, Any]:
197 """Extract the grading fields from the grader's raw response."""
198 if dataset_type.lower() == "browsecomp":
199 # BrowseComp-specific extraction
200 extracted_answer_match = re.search(
201 r"extracted_final_answer:\s*(.*?)(?:\n|$)", grading_response
202 )
203 extracted_answer = (
204 extracted_answer_match.group(1).strip()
205 if extracted_answer_match
206 else "None"
207 )
209 reasoning_match = re.search(
210 r"reasoning:\s*(.*?)(?:\n\n|\ncorrect:|\Z)",
211 grading_response,
212 re.DOTALL,
213 )
214 reasoning = reasoning_match.group(1).strip() if reasoning_match else ""
216 correct_match = re.search(
217 r"correct:\s*(yes|no)", grading_response, re.IGNORECASE
218 )
219 is_correct = (
220 (correct_match.group(1).lower() == "yes")
221 if correct_match
222 else False
223 )
225 confidence_match = re.search(r"confidence:\s*(\d+)", grading_response)
226 confidence = confidence_match.group(1) if confidence_match else "100"
227 else:
228 # SimpleQA extraction
229 extracted_answer_match = re.search(
230 r"Extracted Answer:\s*(.*?)(?:\n|$)", grading_response
231 )
232 extracted_answer = (
233 extracted_answer_match.group(1).strip()
234 if extracted_answer_match
235 else "None"
236 )
238 reasoning_match = re.search(
239 r"Reasoning:\s*(.*?)(?:\nCorrect:|\Z)",
240 grading_response,
241 re.DOTALL,
242 )
243 reasoning = reasoning_match.group(1).strip() if reasoning_match else ""
245 correct_match = re.search(
246 r"Correct:\s*(yes|no)", grading_response, re.IGNORECASE
247 )
248 is_correct = (
249 (correct_match.group(1).lower() == "yes")
250 if correct_match
251 else False
252 )
254 confidence = "100" # SimpleQA doesn't have confidence
256 return {
257 "extracted_by_grader": extracted_answer,
258 "reasoning": reasoning,
259 "is_correct": is_correct,
260 "graded_confidence": confidence,
261 "grader_response": grading_response,
262 }
265def _grading_error_result(exc: Exception) -> Dict[str, Any]:
266 """Build the single-result payload for a failed grading call."""
267 return {
268 "grading_error": str(exc),
269 "is_correct": False,
270 "graded_confidence": "0",
271 "grader_response": f"Grading failed: {exc!s}",
272 }
275def grade_single_result(
276 result_data: Dict[str, Any],
277 dataset_type: str = "simpleqa",
278 evaluation_config: Optional[Dict[str, Any]] = None,
279 settings_snapshot: Optional[Dict[str, Any]] = None,
280) -> Dict[str, Any]:
281 """
282 Grade a single benchmark result using LLM.
284 The synchronous entry point uses ``invoke`` without running the async
285 core on a temporary event loop. Callers already on an event loop can
286 await the corresponding ``_async`` entry point. Resource cleanup uses
287 the shared close helper; it is separate from invocation dispatch.
289 Args:
290 result_data: Dictionary containing result data with keys: id, problem, correct_answer, response, extracted_answer
291 dataset_type: Type of dataset
292 evaluation_config: Optional custom config for evaluation LLM
293 settings_snapshot: Optional settings snapshot for thread-safe access
295 Returns:
296 Dictionary with grading results
297 """
298 # Get evaluation LLM
299 evaluation_llm = get_evaluation_llm(evaluation_config, settings_snapshot)
301 try:
302 grading_prompt = _start_single_grading(result_data, dataset_type)
304 eval_llm_start = time.time()
305 grading_response = _call_grader_llm(evaluation_llm, grading_prompt)
306 _log_grading_call_done(eval_llm_start)
308 return _parse_grading_response(grading_response, dataset_type)
310 except Exception as e:
311 logger.exception("Error grading single result")
312 return _grading_error_result(e)
313 finally:
314 from ..utilities.resource_utils import safe_close
316 safe_close(evaluation_llm, "grader LLM")
319async def grade_single_result_async(
320 result_data: Dict[str, Any],
321 dataset_type: str = "simpleqa",
322 evaluation_config: Optional[Dict[str, Any]] = None,
323 settings_snapshot: Optional[Dict[str, Any]] = None,
324) -> Dict[str, Any]:
325 """Async core of :func:`grade_single_result`: one awaited ainvoke (#5854).
327 For callers that already run on an event loop. Mirrors the sync
328 entry point; only the LLM call line differs.
329 """
330 # Get evaluation LLM
331 evaluation_llm = get_evaluation_llm(evaluation_config, settings_snapshot)
333 try:
334 grading_prompt = _start_single_grading(result_data, dataset_type)
336 eval_llm_start = time.time()
337 grading_response = await _acall_grader_llm(
338 evaluation_llm, grading_prompt
339 )
340 _log_grading_call_done(eval_llm_start)
342 return _parse_grading_response(grading_response, dataset_type)
344 except Exception as e:
345 logger.exception("Error grading single result")
346 return _grading_error_result(e)
347 finally:
348 from ..utilities.resource_utils import safe_close
350 safe_close(evaluation_llm, "grader LLM")
353def _start_single_grading(
354 result_data: Dict[str, Any], dataset_type: str
355) -> str:
356 """Log the start of a single grading call and build its prompt."""
357 template = _select_grader_template(dataset_type)
358 question = result_data.get("problem", "")
359 logger.info(f"Grading single result: {question[:50]}...")
361 grading_prompt = _build_grading_prompt(template, result_data)
362 logger.info(
363 f"Starting grading LLM call (prompt length: {len(grading_prompt)} chars)..."
364 )
365 return grading_prompt
368def _log_grading_call_done(eval_llm_start: float) -> None:
369 """Log how long the grading LLM call took."""
370 eval_llm_elapsed = time.time() - eval_llm_start
371 logger.info(f"Grading LLM call completed in {eval_llm_elapsed:.2f}s")
374def grade_results(
375 results_file: str,
376 output_file: str,
377 dataset_type: str = "simpleqa",
378 evaluation_config: Optional[Dict[str, Any]] = None,
379 progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
380 settings_snapshot: Optional[Dict[str, Any]] = None,
381) -> List[Dict[str, Any]]:
382 """
383 Grade benchmark results using LLM.
385 This synchronous entry point calls the grader through ``invoke``
386 without running the async core on a temporary loop. Shared resource
387 cleanup retains its existing behavior. Callers already on a loop
388 should await :func:`grade_results_async` instead.
390 Args:
391 results_file: Path to results file
392 output_file: Path to save graded results
393 dataset_type: Type of dataset
394 evaluation_config: Optional custom config for evaluation LLM
395 progress_callback: Optional callback for progress updates
396 settings_snapshot: Optional snapshot so the grader LLM is
397 constructed under the user's egress policy. Without it
398 the LLM PEP takes its snapshot-less path, which fails
399 closed with PolicyDeniedError for any non-local provider.
401 Returns:
402 List of graded results
403 """
404 # Get evaluation LLM
405 evaluation_llm = get_evaluation_llm(evaluation_config, settings_snapshot)
407 try:
408 return _grade_results_inner(
409 evaluation_llm,
410 results_file,
411 output_file,
412 dataset_type,
413 progress_callback,
414 )
415 finally:
416 from ..utilities.resource_utils import safe_close
418 safe_close(evaluation_llm, "grader LLM")
421async def grade_results_async(
422 results_file: str,
423 output_file: str,
424 dataset_type: str = "simpleqa",
425 evaluation_config: Optional[Dict[str, Any]] = None,
426 progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
427 settings_snapshot: Optional[Dict[str, Any]] = None,
428) -> List[Dict[str, Any]]:
429 """Async core of :func:`grade_results`: awaited ainvoke calls (#5854).
431 For callers that already run on an event loop.
432 """
433 # Get evaluation LLM
434 evaluation_llm = get_evaluation_llm(evaluation_config, settings_snapshot)
436 try:
437 return await _grade_results_inner_async(
438 evaluation_llm,
439 results_file,
440 output_file,
441 dataset_type,
442 progress_callback,
443 )
444 finally:
445 from ..utilities.resource_utils import safe_close
447 safe_close(evaluation_llm, "grader LLM")
450def _load_results_for_grading(
451 results_file: str, output_file: str
452) -> List[Dict[str, Any]]:
453 """Load the results to grade and clear any previous output file."""
454 results = []
455 with open(results_file, "r", encoding="utf-8") as f:
456 for line in f:
457 if line.strip(): 457 ↛ 456line 457 didn't jump to line 456 because the condition on line 457 was always true
458 results.append(json.loads(line))
460 # Remove output file if it exists
461 output_path = Path(output_file)
462 if output_path.exists():
463 # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python grading API intentionally replaces its caller-selected output file; web grading uses grade_single_result
464 output_path.unlink()
466 return results
469def _append_graded_line(output_file: str, payload: Dict[str, Any]) -> None:
470 """Append one graded (or errored) result to the output file."""
471 with open(output_file, "a", encoding="utf-8") as f:
472 f.write(json.dumps(payload) + "\n")
475def _start_batch_item(
476 template: str,
477 result: Dict[str, Any],
478 idx: int,
479 total: int,
480 progress_callback: Optional[Callable[[int, int, Dict], None]],
481) -> str:
482 """Report progress for one batch item and build its grading prompt."""
483 # Call progress callback if provided
484 if progress_callback:
485 progress_callback(
486 idx,
487 total,
488 {"status": "grading", "index": idx, "total": total},
489 )
491 question = result.get("problem", "")
492 logger.info(f"Grading {idx + 1}/{total}: {question[:50]}...")
494 return _build_grading_prompt(template, result)
497def _finish_batch_item(
498 result: Dict[str, Any],
499 grading_response: str,
500 dataset_type: str,
501 output_file: str,
502 idx: int,
503 total: int,
504 progress_callback: Optional[Callable[[int, int, Dict], None]],
505 graded_results: List[Dict[str, Any]],
506) -> bool:
507 """Parse, record and report one successfully graded batch item."""
508 parsed = _parse_grading_response(grading_response, dataset_type)
509 is_correct = parsed["is_correct"]
511 # Format graded result
512 graded_result = result.copy()
513 graded_result.update(parsed)
515 graded_results.append(graded_result)
517 # Write to output file
518 _append_graded_line(output_file, graded_result)
520 # Call progress callback if provided
521 if progress_callback:
522 progress_callback(
523 idx,
524 total,
525 {
526 "status": "graded",
527 "is_correct": is_correct,
528 "result": graded_result,
529 },
530 )
532 return is_correct
535def _fail_batch_item(
536 result: Dict[str, Any],
537 exc: Exception,
538 output_file: str,
539 idx: int,
540 total: int,
541 progress_callback: Optional[Callable[[int, int, Dict], None]],
542 graded_results: List[Dict[str, Any]],
543) -> None:
544 """Record and report one batch item whose grading raised."""
545 # Handle error
546 error_result = result.copy()
547 error_result["grading_error"] = str(exc)
549 _append_graded_line(output_file, error_result)
551 graded_results.append(error_result)
553 # Call progress callback if provided
554 if progress_callback:
555 progress_callback(
556 idx,
557 total,
558 {
559 "status": "error",
560 "error": str(exc),
561 "result": error_result,
562 },
563 )
566def _log_grading_summary(correct_count: int, total: int) -> None:
567 """Log the final accuracy of a grading run."""
568 accuracy = correct_count / total if total else 0
569 logger.info(f"Grading complete. Accuracy: {accuracy:.3f}")
570 logger.info(f"Correct: {correct_count}/{total}")
573def _grade_results_inner(
574 evaluation_llm,
575 results_file: str,
576 output_file: str,
577 dataset_type: str,
578 progress_callback: Optional[Callable[[int, int, Dict], None]],
579) -> List[Dict[str, Any]]:
580 """Inner implementation of grade_results, separated for cleanup."""
581 template = _select_grader_template(dataset_type)
582 results = _load_results_for_grading(results_file, output_file)
584 graded_results: List[Dict[str, Any]] = []
585 correct_count = 0
586 total = len(results)
588 # Process each result
589 for idx, result in enumerate(results):
590 grading_prompt = _start_batch_item(
591 template, result, idx, total, progress_callback
592 )
594 try:
595 grading_response = _call_grader_llm(evaluation_llm, grading_prompt)
596 if _finish_batch_item(
597 result,
598 grading_response,
599 dataset_type,
600 output_file,
601 idx,
602 total,
603 progress_callback,
604 graded_results,
605 ):
606 correct_count += 1
607 except Exception as e:
608 logger.exception(f"Error grading result {idx + 1}")
609 _fail_batch_item(
610 result,
611 e,
612 output_file,
613 idx,
614 total,
615 progress_callback,
616 graded_results,
617 )
619 _log_grading_summary(correct_count, total)
621 return graded_results
624async def _grade_results_inner_async(
625 evaluation_llm,
626 results_file: str,
627 output_file: str,
628 dataset_type: str,
629 progress_callback: Optional[Callable[[int, int, Dict], None]],
630) -> List[Dict[str, Any]]:
631 """Async counterpart of :func:`_grade_results_inner`.
633 Only the LLM call line differs.
634 """
635 template = _select_grader_template(dataset_type)
636 results = _load_results_for_grading(results_file, output_file)
638 graded_results: List[Dict[str, Any]] = []
639 correct_count = 0
640 total = len(results)
642 # Process each result
643 for idx, result in enumerate(results):
644 grading_prompt = _start_batch_item(
645 template, result, idx, total, progress_callback
646 )
648 try:
649 grading_response = await _acall_grader_llm(
650 evaluation_llm, grading_prompt
651 )
652 if _finish_batch_item( 652 ↛ 643line 652 didn't jump to line 643 because the condition on line 652 was always true
653 result,
654 grading_response,
655 dataset_type,
656 output_file,
657 idx,
658 total,
659 progress_callback,
660 graded_results,
661 ):
662 correct_count += 1
663 except Exception as e:
664 logger.exception(f"Error grading result {idx + 1}")
665 _fail_batch_item(
666 result,
667 e,
668 output_file,
669 idx,
670 total,
671 progress_callback,
672 graded_results,
673 )
675 _log_grading_summary(correct_count, total)
677 return graded_results
680def human_evaluation(
681 results_file: str, output_file: str, interactive: bool = True
682) -> List[Dict[str, Any]]:
683 """
684 Allow for human evaluation of results.
686 Args:
687 results_file: Path to results file
688 output_file: Path to save human-graded results
689 interactive: Whether to run in interactive console mode
691 Returns:
692 List of human-graded results
693 """
694 # Load results
695 results = []
696 with open(results_file, "r", encoding="utf-8") as f:
697 for line in f:
698 if line.strip(): 698 ↛ 697line 698 didn't jump to line 697 because the condition on line 698 was always true
699 results.append(json.loads(line))
701 # Remove output file if it exists
702 output_path = Path(output_file)
703 if output_path.exists():
704 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Interactive local grading intentionally replaces its caller-selected output file
705 output_path.unlink()
707 human_graded_results = []
708 correct_count = 0
710 if interactive: 710 ↛ 711line 710 didn't jump to line 711 because the condition on line 710 was never true
711 logger.info(f"Human evaluation: {len(results)} examples to grade")
712 print(f"Human evaluation: {len(results)} examples to grade")
713 print(
714 "For each example, you'll see the question, correct answer, and model's response."
715 )
716 print("You'll be asked to judge if the model's answer is correct.")
718 for idx, result in enumerate(results):
719 question = result.get("problem", "")
720 correct_answer = result.get("correct_answer", "")
721 response = result.get("response", "")
722 extracted_answer = result.get("extracted_answer", "")
724 if interactive: 724 ↛ 725line 724 didn't jump to line 725 because the condition on line 724 was never true
725 print(f"\n\n===== Example {idx + 1}/{len(results)} =====")
726 print(f"Question: {question}")
727 print(f"\nCorrect Answer: {correct_answer}")
728 print(f"\nModel Response: {response}")
729 print(f"\nExtracted Answer: {extracted_answer}")
731 # Get human judgment
732 while True:
733 judgment = (
734 input("\nIs the model's answer correct? (y/n): ")
735 .strip()
736 .lower()
737 )
738 if judgment in ["y", "n"]:
739 break
740 print("Please enter 'y' or 'n'")
742 is_correct = judgment == "y"
744 # Get reasoning
745 reasoning = input(
746 "Please provide reasoning for your judgment: "
747 ).strip()
748 else:
749 # Non-interactive mode - placeholder for API/UI implementation
750 # In a real implementation, this would be filled by UI actions
751 is_correct = False
752 reasoning = "Non-interactive evaluation"
754 if is_correct: 754 ↛ 755line 754 didn't jump to line 755 because the condition on line 754 was never true
755 correct_count += 1
757 # Update result with human judgment
758 human_result = result.copy()
759 human_result.update(
760 {
761 "is_correct": is_correct,
762 "reasoning": reasoning,
763 "human_evaluation": True,
764 }
765 )
767 human_graded_results.append(human_result)
769 # Write to output file
770 with open(output_file, "a", encoding="utf-8") as f:
771 f.write(json.dumps(human_result) + "\n")
773 accuracy = correct_count / len(results) if results else 0
774 logger.info(f"Human evaluation complete. Accuracy: {accuracy:.3f}")
775 if interactive: 775 ↛ 776line 775 didn't jump to line 776 because the condition on line 775 was never true
776 print(f"\nHuman evaluation complete. Accuracy: {accuracy:.3f}")
777 print(f"Correct: {correct_count}/{len(results)}")
779 return human_graded_results