Coverage for src / local_deep_research / advanced_search_system / questions / browsecomp_question.py: 95%
151 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-14 23:55 +0000
1"""
2BrowseComp-specific question generation that creates progressive, entity-focused searches.
3"""
5import re
6from typing import Dict, List, Optional
8from loguru import logger
10from .base_question import BaseQuestionGenerator
13class BrowseCompQuestionGenerator(BaseQuestionGenerator):
14 """
15 Question generator optimized for BrowseComp-style queries.
17 Key features:
18 1. Extract concrete entities (dates, numbers, names, places)
19 2. Generate progressive search combinations
20 3. Start broad, then narrow systematically
21 4. Focus on verifiable facts
22 """
24 def __init__(
25 self,
26 model,
27 knowledge_truncate_length: int = 1500,
28 previous_searches_limit: int = 10,
29 ):
30 """Initialize the question generator.
32 Args:
33 model: The LLM model to use for generation
34 knowledge_truncate_length: Max chars for knowledge in prompts (None=unlimited)
35 previous_searches_limit: Max previous searches to show (None=unlimited)
36 """
37 super().__init__(model)
38 self.extracted_entities: Dict[str, List[str]] = {}
39 self.search_progression: List[str] = []
40 self.knowledge_truncate_length = knowledge_truncate_length
41 self.previous_searches_limit = previous_searches_limit
43 def generate_questions(
44 self,
45 current_knowledge: str,
46 query: str,
47 questions_per_iteration: int = 5,
48 questions_by_iteration: Optional[dict] = None,
49 results_by_iteration: Optional[dict] = None,
50 iteration: int = 1,
51 ) -> List[str]:
52 """Generate progressive search queries for BrowseComp problems."""
53 questions_by_iteration = questions_by_iteration or {}
55 # First iteration: Extract entities and create initial searches
56 if iteration == 1 or not self.extracted_entities:
57 self.extracted_entities = self._extract_entities(query)
58 return self._generate_initial_searches(
59 query, self.extracted_entities, questions_per_iteration
60 )
62 # Subsequent iterations: Progressive refinement
63 return self._generate_progressive_searches(
64 query,
65 current_knowledge,
66 self.extracted_entities,
67 questions_by_iteration,
68 results_by_iteration or {},
69 questions_per_iteration,
70 iteration,
71 )
73 def _extract_entities(self, query: str) -> Dict[str, List[str]]:
74 """Extract concrete entities from the query."""
75 prompt = f"""Extract ALL concrete, searchable entities from this query:
77Query: {query}
79Extract:
801. TEMPORAL: All years, dates, time periods (e.g., "2018", "between 1995 and 2006", "2023")
812. NUMERICAL: All numbers, statistics, counts (e.g., "300", "more than 3", "4-3", "84.5%")
823. NAMES: Partial names, name hints, proper nouns (e.g., "Dartmouth", "EMNLP", "Plastic Man")
834. LOCATIONS: Places, institutions, geographic features (e.g., "Pennsylvania", "Grand Canyon")
845. DESCRIPTORS: Key descriptive terms (e.g., "fourth wall", "ascetics", "decider game")
86For TEMPORAL entities, if there's a range (e.g., "between 2018-2023"), list EACH individual year.
88Format your response as:
89TEMPORAL: [entity1], [entity2], ...
90NUMERICAL: [entity1], [entity2], ...
91NAMES: [entity1], [entity2], ...
92LOCATIONS: [entity1], [entity2], ...
93DESCRIPTORS: [entity1], [entity2], ...
94"""
96 response = self.model.invoke(prompt)
97 content = (
98 response.content if hasattr(response, "content") else str(response)
99 )
101 entities: Dict[str, List[str]] = {
102 "temporal": [],
103 "numerical": [],
104 "names": [],
105 "locations": [],
106 "descriptors": [],
107 }
109 for line in content.strip().split("\n"):
110 line = line.strip()
111 if ":" in line:
112 category, values = line.split(":", 1)
113 category = category.strip().lower()
114 if category in entities:
115 # Parse comma-separated values
116 values = [v.strip() for v in values.split(",") if v.strip()]
117 entities[category].extend(values)
119 # Expand temporal ranges
120 entities["temporal"] = self._expand_temporal_ranges(
121 entities["temporal"]
122 )
124 logger.info(f"Extracted entities: {entities}")
125 return entities
127 def _expand_temporal_ranges(
128 self, temporal_entities: List[str]
129 ) -> List[str]:
130 """Expand year ranges into individual years."""
131 expanded = []
132 for entity in temporal_entities:
133 # Check for range patterns like "2018-2023" or "between 1995 and 2006"
134 range_match = re.search(
135 r"(\d{4})[-\s]+(?:to|and)?\s*(\d{4})", entity
136 )
137 if range_match:
138 start_year = int(range_match.group(1))
139 end_year = int(range_match.group(2))
140 for year in range(start_year, end_year + 1):
141 expanded.append(str(year))
142 else:
143 # Single year or other temporal entity
144 year_match = re.search(r"\d{4}", entity)
145 if year_match:
146 expanded.append(year_match.group())
147 else:
148 expanded.append(entity)
150 return list(set(expanded)) # Remove duplicates
152 def _generate_initial_searches(
153 self, query: str, entities: Dict[str, List[str]], num_questions: int
154 ) -> List[str]:
155 """Generate initial broad searches."""
156 searches = []
158 # 1. Original query (always include)
159 searches.append(query)
161 # If only 1 question requested, return just the original query
162 if num_questions <= 1:
163 return searches[:1]
165 # 2. Domain exploration searches (combine key entities)
166 if entities["names"] and len(searches) < num_questions:
167 for name in entities["names"][:2]: # Top 2 names
168 if len(searches) >= num_questions:
169 break
170 searches.append(f"{name}")
171 if entities["descriptors"] and len(searches) < num_questions:
172 searches.append(f"{name} {entities['descriptors'][0]}")
174 # 3. Temporal searches if years are important
175 if (
176 entities["temporal"]
177 and len(entities["temporal"]) <= 10
178 and len(searches) < num_questions
179 ):
180 # For small year ranges, search each year with a key term
181 key_term = (
182 entities["names"][0]
183 if entities["names"]
184 else entities["descriptors"][0]
185 if entities["descriptors"]
186 else ""
187 )
188 for year in entities["temporal"][:5]: # Limit to 5 years initially
189 if len(searches) >= num_questions: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 break
191 if key_term: 191 ↛ 188line 191 didn't jump to line 188 because the condition on line 191 was always true
192 searches.append(f"{key_term} {year}")
194 # 4. Location-based searches
195 if entities["locations"] and len(searches) < num_questions:
196 for location in entities["locations"][:2]:
197 if len(searches) >= num_questions: 197 ↛ 198line 197 didn't jump to line 198 because the condition on line 197 was never true
198 break
199 searches.append(f"{location}")
200 if entities["descriptors"] and len(searches) < num_questions:
201 searches.append(f"{location} {entities['descriptors'][0]}")
203 # Remove duplicates and limit to requested number
204 seen = set()
205 unique_searches = []
206 for s in searches:
207 if s.lower() not in seen:
208 seen.add(s.lower())
209 unique_searches.append(s)
210 if len(unique_searches) >= num_questions:
211 break
213 return unique_searches[:num_questions]
215 def _generate_progressive_searches(
216 self,
217 query: str,
218 current_knowledge: str,
219 entities: Dict[str, List[str]],
220 questions_by_iteration: dict,
221 results_by_iteration: dict,
222 num_questions: int,
223 iteration: int,
224 ) -> List[str]:
225 """Generate progressively more specific searches based on findings."""
227 # Only add strategy instructions if we have actual result data (adaptive mode)
228 strategy_instruction = ""
229 if results_by_iteration:
230 # Check if recent searches are failing (returning 0 results)
231 recent_iterations = list(range(max(1, iteration - 5), iteration))
232 zero_count = sum(
233 1
234 for i in recent_iterations
235 if results_by_iteration.get(i, 1) == 0
236 )
237 searches_failing = zero_count >= 3
239 # Adjust strategy based on success/failure
240 if searches_failing:
241 strategy_instruction = """
242IMPORTANT: Your recent searches are returning 0 results - they are TOO NARROW!
243- Use FEWER constraints (1-2 terms instead of 4-5)
244- Try BROADER, more general searches
245- Remove overly specific combinations
246- Focus on key concepts, not detailed entity combinations
247"""
248 else:
249 strategy_instruction = """
250Focus on finding the specific answer by combining entities systematically.
251"""
253 # Analyze what we've found so far
254 prompt = f"""Based on our search progress, generate targeted follow-up searches.
255{strategy_instruction}
257Original Query: {query}
259Entities Found:
260- Names/Terms: {", ".join(entities["names"][:5])}
261- Years: {", ".join(entities["temporal"][:5])}
262- Locations: {", ".join(entities["locations"][:3])}
263- Key Features: {", ".join(entities["descriptors"][:3])}
265Current Knowledge Summary:
266{current_knowledge[: self.knowledge_truncate_length] if self.knowledge_truncate_length else current_knowledge}
268Previous Searches:
269{self._format_previous_searches(questions_by_iteration, results_by_iteration)}
271Generate {num_questions} NEW search queries that:
2721. Combine 2-3 entities we haven't tried together
2732. If we found candidate names, search for them with other constraints
2743. For year ranges, systematically cover years we haven't searched
2754. Use quotes for exact phrases when beneficial
277Focus on finding the specific answer, not general information.
279Format: One search per line
280"""
282 response = self.model.invoke(prompt)
283 content = (
284 response.content if hasattr(response, "content") else str(response)
285 )
287 # Extract searches from response
288 searches = []
289 for line in content.strip().split("\n"):
290 line = line.strip()
291 if line and not line.endswith(":") and len(line) > 5: 291 ↛ 289line 291 didn't jump to line 289 because the condition on line 291 was always true
292 # Clean up common prefixes
293 for prefix in ["Q:", "Search:", "-", "*", "•"]:
294 if line.startswith(prefix):
295 line = line[len(prefix) :].strip()
296 if line: 296 ↛ 289line 296 didn't jump to line 289 because the condition on line 296 was always true
297 searches.append(line)
299 # Ensure we have enough searches, but respect the limit
300 while len(searches) < num_questions:
301 # Generate combinations programmatically
302 if iteration <= 5 and entities["temporal"]:
303 # Continue with year-based searches
304 added_any = False
305 for year in entities["temporal"]: 305 ↛ 314line 305 didn't jump to line 314 because the loop on line 305 didn't complete
306 if not self._was_searched(year, questions_by_iteration): 306 ↛ 305line 306 didn't jump to line 305 because the condition on line 306 was always true
307 base_term = (
308 entities["names"][0] if entities["names"] else ""
309 )
310 searches.append(f"{base_term} {year}".strip())
311 added_any = True
312 if len(searches) >= num_questions:
313 break
314 if not added_any: 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true
315 break # No more year searches to add
316 else:
317 # Combine multiple constraints
318 added_any = False
319 if entities["names"] and entities["descriptors"]:
320 for name in entities["names"]:
321 for desc in entities["descriptors"]:
322 combo = f"{name} {desc}"
323 if not self._was_searched(
324 combo, questions_by_iteration
325 ):
326 searches.append(combo)
327 added_any = True
328 if len(searches) >= num_questions:
329 break
330 if len(searches) >= num_questions:
331 break
332 if not added_any:
333 break # No more combinations to add
335 return searches[:num_questions]
337 def _format_previous_searches(
338 self, questions_by_iteration: dict, results_by_iteration: dict
339 ) -> str:
340 """Format previous searches for context with result counts."""
341 formatted = []
342 for iteration, questions in questions_by_iteration.items():
343 if isinstance(questions, list): 343 ↛ 342line 343 didn't jump to line 342 because the condition on line 343 was always true
344 result_count = results_by_iteration.get(iteration, "?")
345 # Limit questions per iteration (main uses 3)
346 questions_to_show = (
347 questions[:3] if self.previous_searches_limit else questions
348 )
349 for q in questions_to_show:
350 # Only show result counts if we have actual data (not "?")
351 if result_count != "?":
352 formatted.append(
353 f"Iteration {iteration}: {q} ({result_count} results)"
354 )
355 else:
356 formatted.append(f"Iteration {iteration}: {q}")
357 # Apply limit if configured (main uses last 10)
358 if self.previous_searches_limit:
359 return "\n".join(formatted[-self.previous_searches_limit :])
360 return "\n".join(formatted)
362 def _was_searched(self, term: str, questions_by_iteration: dict) -> bool:
363 """Check if a term was already searched."""
364 term_lower = term.lower()
365 for questions in questions_by_iteration.values():
366 if isinstance(questions, list):
367 for q in questions:
368 if term_lower in q.lower():
369 return True
370 return False