Coverage for src/local_deep_research/benchmarks/graders.py: 89%

1"""

2Evaluation and grading functionality.

4This module provides tools for evaluating model outputs against reference answers.

5"""

7import json

8from loguru import logger

9from pathlib import Path

10import re

11from typing import Any, Callable, Dict, List, Optional

13from langchain_core.messages.human import HumanMessage

15from ..config.llm_config import get_llm

16from ..llm.providers.base import normalize_provider

17from .templates import BROWSECOMP_GRADER_TEMPLATE, SIMPLEQA_GRADER_TEMPLATE

20# Default evaluation configuration using Claude 3.7 Sonnet via OpenRouter

21DEFAULT_EVALUATION_CONFIG = {

22 "model_name": "anthropic/claude-3.7-sonnet", # Correct model ID for OpenRouter

23 "provider": "openai_endpoint", # Use OpenRouter

24 "openai_endpoint_url": "https://openrouter.ai/api/v1", # OpenRouter URL

25 "temperature": 0, # Zero temp for consistent evaluation

26 # Note: max_tokens removed as it's not supported by LDR's get_llm()

27}

30def get_evaluation_llm(

31 custom_config: Optional[Dict[str, Any]] = None,

32 settings_snapshot: Optional[Dict[str, Any]] = None,

33):

34 """

35 Get an LLM for evaluation purposes using Claude 3.7 Sonnet via OpenRouter

36 by default, which can be overridden with custom settings.

38 Args:

39 custom_config: Optional custom configuration that overrides defaults

40 settings_snapshot: Optional settings snapshot for thread-safe access

42 Returns:

43 An LLM instance for evaluation

44 """

45 # Start with default config (Claude 3.7 Sonnet via OpenRouter)

46 config = DEFAULT_EVALUATION_CONFIG.copy()

48 # Override with any custom settings

49 if custom_config:

50 config.update(custom_config)

52 logger.info(

53 f"Getting evaluation LLM with provider={config['provider']}, model={config['model_name']}"

54 )

56 # Remove any parameters that LDR's get_llm doesn't support

57 # This ensures compatibility with LDR's implementation

58 ldr_supported_params = {

59 "model_name",

60 "temperature",

61 "provider",

62 "openai_endpoint_url",

63 "api_key",

64 }

66 filtered_config = {

67 k: v for k, v in config.items() if k in ldr_supported_params

68 }

70 # Check if we're using openai_endpoint but don't have an API key configured

71 if normalize_provider(filtered_config.get("provider")) == "openai_endpoint":

72 # Try to get API key from settings snapshot or environment

73 api_key = None

75 if settings_snapshot:

76 # Get from settings snapshot for thread safety

77 api_key_setting = settings_snapshot.get(

78 "llm.openai_endpoint.api_key"

79 )

80 if api_key_setting:

81 api_key = (

82 api_key_setting.get("value")

83 if isinstance(api_key_setting, dict)

84 else api_key_setting

85 )

86 else:

87 # No settings snapshot available

88 logger.warning(

89 "No settings snapshot provided for benchmark grader. "

90 "API key must be provided via settings_snapshot for thread safety."

91 )

93 if not api_key:

94 logger.warning(

95 "Using openai_endpoint provider but no API key found. "

96 "Set the llm.openai_endpoint.api_key setting in the database or "

97 "LDR_LLM_OPENAI_ENDPOINT_API_KEY environment variable."

98 )

99 # Try to fall back to LDR's config if API key not explicitly provided

100 # The get_llm function will handle this case

101

102 # Get the LLM using LDR's existing function. Thread settings_snapshot

103 # through — without it, get_llm's snapshot-less PEP only permits local

104 # default providers, so a cloud grader (e.g. openai) would be refused

105 # with PolicyDeniedError. The snapshot lets get_llm evaluate the policy

106 # and permit the configured grader when the user's scope allows it.

107 return get_llm(**filtered_config, settings_snapshot=settings_snapshot)

108

109

110def extract_answer_from_response(

111 response: str, dataset_type: str = "simpleqa"

112) -> Dict[str, str]:

113 """

114 Extract structured information from LDR's response.

115

116 Args:

117 response: Response from LDR

118 dataset_type: Type of dataset

119

120 Returns:

121 Dictionary with extracted answer and confidence

122 """

123 # Clean up citations — strip both ASCII "[N]" and lenticular "【N】"

124 # so a lenticular citation (some LLMs emit them) doesn't survive into

125 # the graded answer text and skew the match.

126 response = re.sub(r"[\[【]\d+[\]】]", "", response)

127

128 # Extract differently based on dataset type

129 if dataset_type.lower() == "browsecomp":

130 # Extract the final answer from structured response

131 answer_match = re.search(r"Exact Answer:\s*(.*?)(?:\n|$)", response)

132 exact_answer = answer_match.group(1).strip() if answer_match else "None"

133

134 # Extract confidence

135 confidence_match = re.search(r"Confidence:\s*(\d+)%", response)

136 confidence = confidence_match.group(1) if confidence_match else "100"

137

138 return {"extracted_answer": exact_answer, "confidence": confidence}

139

140 # For SimpleQA, return the whole response as the answer

141 return {

142 "extracted_answer": response,

143 "confidence": "100", # SimpleQA doesn't have confidence scores

144 }

145

146

147def grade_single_result(

148 result_data: Dict[str, Any],

149 dataset_type: str = "simpleqa",

150 evaluation_config: Optional[Dict[str, Any]] = None,

151 settings_snapshot: Optional[Dict[str, Any]] = None,

152) -> Dict[str, Any]:

153 """

154 Grade a single benchmark result using LLM.

155

156 Args:

157 result_data: Dictionary containing result data with keys: id, problem, correct_answer, response, extracted_answer

158 dataset_type: Type of dataset

159 evaluation_config: Optional custom config for evaluation LLM

160 settings_snapshot: Optional settings snapshot for thread-safe access

161

162 Returns:

163 Dictionary with grading results

164 """

165 # Get evaluation LLM

166 evaluation_llm = get_evaluation_llm(evaluation_config, settings_snapshot)

167

168 try:

169 # Select appropriate template

170 template = (

171 BROWSECOMP_GRADER_TEMPLATE

172 if dataset_type.lower() == "browsecomp"

173 else SIMPLEQA_GRADER_TEMPLATE

174 )

175

176 question = result_data.get("problem", "")

177 correct_answer = result_data.get("correct_answer", "")

178 response = result_data.get("response", "")

179

180 logger.info(f"Grading single result: {question[:50]}...")

181

182 # Format grading prompt

183 grading_prompt = template.format(

184 question=question, correct_answer=correct_answer, response=response

185 )

186

187 import time

188

189 eval_llm_start = time.time()

190 logger.info(

191 f"Starting grading LLM call (prompt length: {len(grading_prompt)} chars)..."

192 )

193

194 # Grade using LLM

195 if hasattr(evaluation_llm, "invoke") and callable(

196 evaluation_llm.invoke

197 ):

198 if hasattr(evaluation_llm, "chat_messages"):

199 # Handle ChatOpenAI and similar models that use messages

200 grading_response = evaluation_llm.invoke(

201 [HumanMessage(content=grading_prompt)]

202 ).content

203 else:

204 # Handle other LLM types

205 grading_response = evaluation_llm.invoke(grading_prompt)

206 if hasattr(grading_response, "content"):

207 grading_response = grading_response.content

208 else:

209 # Fallback for other LLM interfaces

210 grading_response = str(evaluation_llm(grading_prompt))

211

212 eval_llm_elapsed = time.time() - eval_llm_start

213 logger.info(f"Grading LLM call completed in {eval_llm_elapsed:.2f}s")

214

215 # Extract grading information using regex

216 if dataset_type.lower() == "browsecomp":

217 # BrowseComp-specific extraction

218 extracted_answer_match = re.search(

219 r"extracted_final_answer:\s*(.*?)(?:\n|$)", grading_response

220 )

221 extracted_answer = (

222 extracted_answer_match.group(1).strip()

223 if extracted_answer_match

224 else "None"

225 )

226

227 reasoning_match = re.search(

228 r"reasoning:\s*(.*?)(?:\n\n|\ncorrect:|\Z)",

229 grading_response,

230 re.DOTALL,

231 )

232 reasoning = (

233 reasoning_match.group(1).strip() if reasoning_match else ""

234 )

235

236 correct_match = re.search(

237 r"correct:\s*(yes|no)", grading_response, re.IGNORECASE

238 )

239 is_correct = (

240 (correct_match.group(1).lower() == "yes")

241 if correct_match

242 else False

243 )

244

245 confidence_match = re.search(

246 r"confidence:\s*(\d+)", grading_response

247 )

248 confidence = (

249 confidence_match.group(1) if confidence_match else "100"

250 )

251 else:

252 # SimpleQA extraction

253 extracted_answer_match = re.search(

254 r"Extracted Answer:\s*(.*?)(?:\n|$)", grading_response

255 )

256 extracted_answer = (

257 extracted_answer_match.group(1).strip()

258 if extracted_answer_match

259 else "None"

260 )

261

262 reasoning_match = re.search(

263 r"Reasoning:\s*(.*?)(?:\nCorrect:|\Z)",

264 grading_response,

265 re.DOTALL,

266 )

267 reasoning = (

268 reasoning_match.group(1).strip() if reasoning_match else ""

269 )

270

271 correct_match = re.search(

272 r"Correct:\s*(yes|no)", grading_response, re.IGNORECASE

273 )

274 is_correct = (

275 (correct_match.group(1).lower() == "yes")

276 if correct_match

277 else False

278 )

279

280 confidence = "100" # SimpleQA doesn't have confidence

281

282 # Format graded result

283 return {

284 "extracted_by_grader": extracted_answer,

285 "reasoning": reasoning,

286 "is_correct": is_correct,

287 "graded_confidence": confidence,

288 "grader_response": grading_response,

289 }

290

291 except Exception as e:

292 logger.exception("Error grading single result")

293 return {

294 "grading_error": str(e),

295 "is_correct": False,

296 "graded_confidence": "0",

297 "grader_response": f"Grading failed: {e!s}",

298 }

299 finally:

300 from ..utilities.resource_utils import safe_close

301

302 safe_close(evaluation_llm, "grader LLM")

303

304

305def grade_results(

306 results_file: str,

307 output_file: str,

308 dataset_type: str = "simpleqa",

309 evaluation_config: Optional[Dict[str, Any]] = None,

310 progress_callback: Optional[Callable[[int, int, Dict], None]] = None,

311 settings_snapshot: Optional[Dict[str, Any]] = None,

312) -> List[Dict[str, Any]]:

313 """

314 Grade benchmark results using LLM.

315

316 Args:

317 results_file: Path to results file

318 output_file: Path to save graded results

319 dataset_type: Type of dataset

320 evaluation_config: Optional custom config for evaluation LLM

321 progress_callback: Optional callback for progress updates

322 settings_snapshot: Optional snapshot so the grader LLM is

323 constructed under the user's egress policy. Without it

324 the LLM PEP's snapshot guard skips and the grader can

325 silently reach a cloud LLM under require_local_endpoint.

326

327 Returns:

328 List of graded results

329 """

330 # Get evaluation LLM

331 evaluation_llm = get_evaluation_llm(evaluation_config, settings_snapshot)

332

333 try:

334 return _grade_results_inner(

335 evaluation_llm,

336 results_file,

337 output_file,

338 dataset_type,

339 progress_callback,

340 )

341 finally:

342 from ..utilities.resource_utils import safe_close

343

344 safe_close(evaluation_llm, "grader LLM")

345

346

347def _grade_results_inner(

348 evaluation_llm,

349 results_file: str,

350 output_file: str,

351 dataset_type: str,

352 progress_callback: Optional[Callable[[int, int, Dict], None]],

353) -> List[Dict[str, Any]]:

354 """Inner implementation of grade_results, separated for cleanup."""

355 # Select appropriate template

356 template = (

357 BROWSECOMP_GRADER_TEMPLATE

358 if dataset_type.lower() == "browsecomp"

359 else SIMPLEQA_GRADER_TEMPLATE

360 )

361

362 # Load results

363 results = []

364 with open(results_file, "r", encoding="utf-8") as f:

365 for line in f:

366 if line.strip(): 366 ↛ 365line 366 didn't jump to line 365 because the condition on line 366 was always true

367 results.append(json.loads(line))

368

369 # Remove output file if it exists

370 output_path = Path(output_file)

371 if output_path.exists():

372 output_path.unlink()

373

374 graded_results = []

375 correct_count = 0

376

377 # Process each result

378 for idx, result in enumerate(results):

379 question = result.get("problem", "")

380 correct_answer = result.get("correct_answer", "")

381 response = result.get("response", "")

382

383 # Call progress callback if provided

384 if progress_callback:

385 progress_callback(

386 idx,

387 len(results),

388 {"status": "grading", "index": idx, "total": len(results)},

389 )

390

391 logger.info(f"Grading {idx + 1}/{len(results)}: {question[:50]}...")

392

393 # Format grading prompt

394 grading_prompt = template.format(

395 question=question, correct_answer=correct_answer, response=response

396 )

397

398 try:

399 # Grade using LLM

400 if hasattr(evaluation_llm, "invoke") and callable(

401 evaluation_llm.invoke

402 ):

403 if hasattr(evaluation_llm, "chat_messages"):

404 # Handle ChatOpenAI and similar models that use messages

405 grading_response = evaluation_llm.invoke(

406 [HumanMessage(content=grading_prompt)]

407 ).content

408 else:

409 # Handle other LLM types

410 grading_response = evaluation_llm.invoke(grading_prompt)

411 if hasattr(grading_response, "content"):

412 grading_response = grading_response.content

413 else:

414 # Fallback for other LLM interfaces

415 grading_response = str(evaluation_llm(grading_prompt))

416

417 # Extract grading information using regex

418 if dataset_type.lower() == "browsecomp":

419 # BrowseComp-specific extraction

420 extracted_answer_match = re.search(

421 r"extracted_final_answer:\s*(.*?)(?:\n|$)", grading_response

422 )

423 extracted_answer = (

424 extracted_answer_match.group(1).strip()

425 if extracted_answer_match

426 else "None"

427 )

428

429 reasoning_match = re.search(

430 r"reasoning:\s*(.*?)(?:\n\n|\ncorrect:|\Z)",

431 grading_response,

432 re.DOTALL,

433 )

434 reasoning = (

435 reasoning_match.group(1).strip() if reasoning_match else ""

436 )

437

438 correct_match = re.search(

439 r"correct:\s*(yes|no)", grading_response, re.IGNORECASE

440 )

441 is_correct = (

442 (correct_match.group(1).lower() == "yes")

443 if correct_match

444 else False

445 )

446

447 confidence_match = re.search(

448 r"confidence:\s*(\d+)", grading_response

449 )

450 confidence = (

451 confidence_match.group(1) if confidence_match else "100"

452 )

453 else:

454 # SimpleQA extraction

455 extracted_answer_match = re.search(

456 r"Extracted Answer:\s*(.*?)(?:\n|$)", grading_response

457 )

458 extracted_answer = (

459 extracted_answer_match.group(1).strip()

460 if extracted_answer_match

461 else "None"

462 )

463

464 reasoning_match = re.search(

465 r"Reasoning:\s*(.*?)(?:\nCorrect:|\Z)",

466 grading_response,

467 re.DOTALL,

468 )

469 reasoning = (

470 reasoning_match.group(1).strip() if reasoning_match else ""

471 )

472

473 correct_match = re.search(

474 r"Correct:\s*(yes|no)", grading_response, re.IGNORECASE

475 )

476 is_correct = (

477 (correct_match.group(1).lower() == "yes")

478 if correct_match

479 else False

480 )

481

482 confidence = "100" # SimpleQA doesn't have confidence

483

484 if is_correct:

485 correct_count += 1

486

487 # Format graded result

488 graded_result = result.copy()

489 graded_result.update(

490 {

491 "extracted_by_grader": extracted_answer,

492 "reasoning": reasoning,

493 "is_correct": is_correct,

494 "graded_confidence": confidence,

495 "grader_response": grading_response,

496 }

497 )

498

499 graded_results.append(graded_result)

500

501 # Write to output file

502 with open(output_file, "a", encoding="utf-8") as f:

503 f.write(json.dumps(graded_result) + "\n")

504

505 # Call progress callback if provided

506 if progress_callback:

507 progress_callback(

508 idx,

509 len(results),

510 {

511 "status": "graded",

512 "is_correct": is_correct,

513 "result": graded_result,

514 },

515 )

516

517 except Exception as e:

518 logger.exception(f"Error grading result {idx + 1}")

519

520 # Handle error

521 error_result = result.copy()

522 error_result["grading_error"] = str(e)

523

524 with open(output_file, "a", encoding="utf-8") as f:

525 f.write(json.dumps(error_result) + "\n")

526

527 graded_results.append(error_result)

528

529 # Call progress callback if provided

530 if progress_callback:

531 progress_callback(

532 idx,

533 len(results),

534 {

535 "status": "error",

536 "error": str(e),

537 "result": error_result,

538 },

539 )

540

541 accuracy = correct_count / len(results) if results else 0

542 logger.info(f"Grading complete. Accuracy: {accuracy:.3f}")

543 logger.info(f"Correct: {correct_count}/{len(results)}")

544

545 return graded_results

546

547

548def human_evaluation(

549 results_file: str, output_file: str, interactive: bool = True

550) -> List[Dict[str, Any]]:

551 """

552 Allow for human evaluation of results.

553

554 Args:

555 results_file: Path to results file

556 output_file: Path to save human-graded results

557 interactive: Whether to run in interactive console mode

558

559 Returns:

560 List of human-graded results

561 """

562 # Load results

563 results = []

564 with open(results_file, "r", encoding="utf-8") as f:

565 for line in f:

566 if line.strip(): 566 ↛ 565line 566 didn't jump to line 565 because the condition on line 566 was always true

567 results.append(json.loads(line))

568

569 # Remove output file if it exists

570 output_path = Path(output_file)

571 if output_path.exists(): 571 ↛ 572line 571 didn't jump to line 572 because the condition on line 571 was never true

572 output_path.unlink()

573

574 human_graded_results = []

575 correct_count = 0

576

577 if interactive: 577 ↛ 578line 577 didn't jump to line 578 because the condition on line 577 was never true

578 logger.info(f"Human evaluation: {len(results)} examples to grade")

579 print(f"Human evaluation: {len(results)} examples to grade")

580 print(

581 "For each example, you'll see the question, correct answer, and model's response."

582 )

583 print("You'll be asked to judge if the model's answer is correct.")

584

585 for idx, result in enumerate(results):

586 question = result.get("problem", "")

587 correct_answer = result.get("correct_answer", "")

588 response = result.get("response", "")

589 extracted_answer = result.get("extracted_answer", "")

590

591 if interactive: 591 ↛ 592line 591 didn't jump to line 592 because the condition on line 591 was never true

592 print(f"\n\n===== Example {idx + 1}/{len(results)} =====")

593 print(f"Question: {question}")

594 print(f"\nCorrect Answer: {correct_answer}")

595 print(f"\nModel Response: {response}")

596 print(f"\nExtracted Answer: {extracted_answer}")

597

598 # Get human judgment

599 while True:

600 judgment = (

601 input("\nIs the model's answer correct? (y/n): ")

602 .strip()

603 .lower()

604 )

605 if judgment in ["y", "n"]:

606 break

607 print("Please enter 'y' or 'n'")

608

609 is_correct = judgment == "y"

610

611 # Get reasoning

612 reasoning = input(

613 "Please provide reasoning for your judgment: "

614 ).strip()

615 else:

616 # Non-interactive mode - placeholder for API/UI implementation

617 # In a real implementation, this would be filled by UI actions

618 is_correct = False

619 reasoning = "Non-interactive evaluation"

620

621 if is_correct: 621 ↛ 622line 621 didn't jump to line 622 because the condition on line 621 was never true

622 correct_count += 1

623

624 # Update result with human judgment

625 human_result = result.copy()

626 human_result.update(

627 {

628 "is_correct": is_correct,

629 "reasoning": reasoning,

630 "human_evaluation": True,

631 }

632 )

633

634 human_graded_results.append(human_result)

635

636 # Write to output file

637 with open(output_file, "a", encoding="utf-8") as f:

638 f.write(json.dumps(human_result) + "\n")

639

640 accuracy = correct_count / len(results) if results else 0

641 logger.info(f"Human evaluation complete. Accuracy: {accuracy:.3f}")

642 if interactive: 642 ↛ 643line 642 didn't jump to line 643 because the condition on line 642 was never true

643 print(f"\nHuman evaluation complete. Accuracy: {accuracy:.3f}")

644 print(f"Correct: {correct_count}/{len(results)}")

645

646 return human_graded_results