Coverage for src/local_deep_research/web_search_engines/search_engine_factory.py: 90%
256 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1import inspect
2from typing import Any, Dict, Optional
4from ..security.egress.policy import PolicyDeniedError
5from ..security.log_sanitizer import scrub_error
6from ..security.module_whitelist import get_safe_module_class
7from ..security.secure_logging import logger
8from .retriever_registry import retriever_registry
9from .search_engine_base import BaseSearchEngine
10from .search_engines_config import search_config
12# Engines whose constructors accept the global ``search.time_period`` filter.
13# Used both by get_search()'s explicit forwarding and by
14# create_search_engine()'s settings-snapshot fallback so that direct callers
15# (langgraph-agent strategy tools, MCP server) honor the setting too.
16TIME_PERIOD_ENGINES = ("serpapi", "tavily", "wikinews")
19def create_search_engine(
20 engine_name: str,
21 llm=None,
22 username: str | None = None,
23 settings_snapshot: Dict[str, Any] | None = None,
24 programmatic_mode: bool = False,
25 **kwargs,
26) -> Optional[BaseSearchEngine]:
27 """
28 Create a search engine instance based on the engine name.
30 Args:
31 engine_name: Name of the search engine to create
32 llm: Language model instance (passed to engines that accept one)
33 programmatic_mode: If True, disables database operations and metrics tracking
34 **kwargs: Additional parameters to override defaults
36 Returns:
37 Initialized search engine instance or None if creation failed
38 """
39 # Debug logging
40 logger.info(
41 f"create_search_engine called with engine_name={engine_name} (type: {type(engine_name)})"
42 )
44 # Resolve the effective username for per-user registry lookups. Callers
45 # that thread it explicitly win; otherwise fall back to the ``_username``
46 # carried in the settings snapshot (injected by search_config.get_search /
47 # AdvancedSearchSystem). None resolves the shared namespace only.
48 if username is None: 48 ↛ 60line 48 didn't jump to line 60 because the condition on line 48 was always true
49 from ..search_system import username_from_snapshot
51 username = username_from_snapshot(settings_snapshot)
53 # Egress policy PEP runs AFTER retriever-registry + config lookup, so
54 # registered retrievers don't go through engine-name classification.
55 # The actual call site is right before engine instantiation, further
56 # down.
58 # Check if this is a registered retriever first (the caller's own
59 # retrievers resolve before shared/global ones).
60 retriever = retriever_registry.get(engine_name, username=username)
61 if retriever:
62 # Egress policy: gate the retriever against the active scope.
63 # Retrievers are classified at registration (is_local, default
64 # True). Snapshot REQUIRED — previously the policy block was
65 # gated on "if settings_snapshot:" which let snapshot=None
66 # callers instantiate retrievers ungated.
67 # NB: PolicyDeniedError is imported at MODULE level — importing it
68 # again here (function-local) would shadow it across the whole
69 # function and make the outer `except PolicyDeniedError` an unbound
70 # local on code paths that skip this branch.
71 from ..security.egress.policy import (
72 Decision,
73 context_from_snapshot,
74 evaluate_retriever,
75 resolve_run_primary_engine,
76 )
78 if not settings_snapshot: 78 ↛ 79line 78 didn't jump to line 79 because the condition on line 78 was never true
79 raise PolicyDeniedError(
80 Decision(False, "no_snapshot"),
81 target=engine_name,
82 )
84 primary = resolve_run_primary_engine(
85 settings_snapshot, default=engine_name
86 )
87 try:
88 _ctx = context_from_snapshot(
89 settings_snapshot,
90 primary,
91 username=username,
92 )
93 except PolicyDeniedError:
94 raise
95 except ValueError as exc:
96 raise PolicyDeniedError(
97 Decision(False, "invalid_policy_config"),
98 target=engine_name,
99 ) from exc
100 # Pass metadata from the SAME registry reference used for
101 # .get() above, so evaluate_retriever doesn't re-read a
102 # different (e.g. test-patched) global singleton.
103 try:
104 _meta = retriever_registry.get_metadata(
105 engine_name, username=username
106 )
107 except AttributeError:
108 _meta = None
109 _decision = evaluate_retriever(engine_name, _ctx, metadata=_meta)
110 if not _decision.allowed: 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true
111 logger.bind(policy_audit=True).warning(
112 "retriever denied by egress policy",
113 retriever=engine_name,
114 scope=_ctx.scope.value,
115 reason=_decision.reason,
116 )
117 raise PolicyDeniedError(_decision, target=engine_name)
119 logger.info(f"Using registered LangChain retriever: {engine_name}")
120 from .engines.search_engine_retriever import RetrieverSearchEngine
122 return RetrieverSearchEngine(
123 retriever=retriever,
124 name=engine_name,
125 max_results=kwargs.get("max_results", 10),
126 programmatic_mode=programmatic_mode,
127 )
129 # Extract search engine configs from settings snapshot
130 if settings_snapshot:
131 config = search_config(
132 username=username, settings_snapshot=settings_snapshot
133 )
135 logger.debug(
136 f"Extracted search engines from snapshot: {list(config.keys())}"
137 )
138 else:
139 raise RuntimeError(
140 "settings_snapshot is required for search engine creation in threads"
141 )
143 if engine_name == "none":
144 # Reject the literal string "none". Historically this silently fell
145 # through to a meta engine and hit live networks — callers that
146 # wanted an offline pipeline were unknowingly doing real searches.
147 raise ValueError(
148 "search.tool='none' is not a valid engine. Register a LangChain "
149 "retriever via `retrievers={...}` (see "
150 "examples/llm_integration/mock_llm_example.py) or pick a real "
151 "engine."
152 )
154 if engine_name not in config:
155 # Deprecated path: some older callers pass UI display labels instead of
156 # config keys. Display labels look like "{icon} {base_name} ({category})"
157 # e.g. "🔬 OpenAlex (Scientific)". Only attempt (and warn about)
158 # this fallback when the name actually matches that shape — plain keys
159 # like "collection_<uuid>" never will, and the warning was pure noise.
160 looks_like_display_label = " (" in engine_name and engine_name.endswith(
161 ")"
162 )
163 if looks_like_display_label:
164 logger.warning(
165 f"Engine '{engine_name}' not found in config - attempting "
166 "display label fallback. This is deprecated; callers should "
167 "pass the config key directly."
168 )
170 # To avoid ReDoS, use string ops instead of regex.
171 # Pattern: icon, space, base_name, space, (category)
172 parts = engine_name.rsplit(" (", 1)
173 if len(parts) == 2: 173 ↛ 201line 173 didn't jump to line 201 because the condition on line 173 was always true
174 before_paren = parts[0]
175 space_idx = before_paren.find(" ")
176 if space_idx > 0:
177 base_name = before_paren[space_idx + 1 :].strip()
178 logger.info(
179 f"Extracted base name '{base_name}' from label "
180 f"'{engine_name}'"
181 )
183 for config_key, config_data in config.items():
184 if isinstance(config_data, dict): 184 ↛ 183line 184 didn't jump to line 183 because the condition on line 184 was always true
185 display_name = config_data.get(
186 "display_name", config_key
187 )
188 if display_name == base_name:
189 logger.info(
190 f"Matched label to config key: "
191 f"'{engine_name}' -> '{config_key}'"
192 )
193 engine_name = config_key
194 break
196 # If still not found, FAIL CLOSED.
197 # We raise a plain ValueError (not PolicyDeniedError) because
198 # "unknown engine name" is a config/wiring error, not a policy
199 # decision — keeping the exception type clean lets policy-aware
200 # callers tell them apart.
201 if engine_name not in config:
202 logger.bind(policy_audit=True).warning(
203 "unknown engine name rejected",
204 engine=engine_name,
205 available=list(config.keys()),
206 )
207 raise ValueError(
208 f"Unknown search engine '{engine_name}'. Available: "
209 f"{sorted(config.keys())}. The 'auto', 'meta', 'parallel' "
210 "and 'parallel_scientific' meta engines were removed — the "
211 "langgraph-agent strategy (the default) selects engines "
212 "dynamically; pick a concrete engine instead."
213 )
215 # Egress policy PEP: gate the resolved engine against the user's declared
216 # scope. Lazy import to break the circular dependency
217 # (security/egress/policy.py → web_search_engines/* → security/*).
218 # PolicyDeniedError comes from the module-level import (see the note
219 # in the retriever branch above — re-importing it here would shadow
220 # it function-wide and break the outer except clause).
221 from ..security.egress.policy import (
222 Decision,
223 context_from_snapshot,
224 evaluate_engine,
225 resolve_run_primary_engine,
226 )
228 # Resolve the run's primary via the shared helper so the scope this PEP
229 # enforces matches the one the LangGraph tool-list filter pre-filters with
230 # (default: this engine when search.tool is unset).
231 primary_engine = resolve_run_primary_engine(
232 settings_snapshot, default=engine_name
233 )
235 try:
236 ctx = context_from_snapshot(
237 settings_snapshot, primary_engine, username=username
238 )
239 except ValueError as exc:
240 raise PolicyDeniedError(
241 Decision(False, "invalid_policy_config"),
242 target=engine_name,
243 ) from exc
245 # Pass the engine config entry as metadata so a per-collection
246 # is_public classification is honored without a redundant DB lookup.
247 decision = evaluate_engine(
248 engine_name,
249 ctx,
250 settings_snapshot=settings_snapshot,
251 metadata=config.get(engine_name),
252 )
253 if not decision.allowed:
254 logger.bind(policy_audit=True).warning(
255 "engine denied by egress policy",
256 engine=engine_name,
257 scope=ctx.scope.value,
258 reason=decision.reason,
259 )
260 raise PolicyDeniedError(decision, target=engine_name)
262 # Get engine configuration
263 engine_config = config[engine_name]
265 # Set default max_results from config if not provided in kwargs
266 if "max_results" not in kwargs:
267 if settings_snapshot and "search.max_results" in settings_snapshot:
268 max_results = (
269 settings_snapshot["search.max_results"].get("value", 20)
270 if isinstance(settings_snapshot["search.max_results"], dict)
271 else settings_snapshot["search.max_results"]
272 )
273 else:
274 max_results = 20
275 kwargs["max_results"] = max_results
277 # Route the global time filter to engines that understand it when the
278 # caller didn't pass one explicitly. get_search() forwards it as a kwarg,
279 # but direct callers (langgraph-agent strategy tools, the MCP server)
280 # only pass a settings_snapshot — without this fallback the user's
281 # Settings → Search → Time Period choice is silently ignored on those
282 # paths. Placed in kwargs so it takes precedence over per-engine
283 # default_params, matching get_search()'s behavior.
284 if (
285 engine_name in TIME_PERIOD_ENGINES
286 and "time_period" not in kwargs
287 and settings_snapshot
288 and "search.time_period" in settings_snapshot
289 ):
290 raw_time_period = settings_snapshot["search.time_period"]
291 kwargs["time_period"] = (
292 raw_time_period.get("value", "all")
293 if isinstance(raw_time_period, dict)
294 else raw_time_period
295 )
297 # Check for API key requirements
298 requires_api_key = engine_config.get("requires_api_key", False)
300 # Bound here (not just inside ``if requires_api_key``) so the except
301 # handler below can pass the in-scope key to scrub_error() — without
302 # the hoist, the no-key path would hit a NameError when scrubbing.
303 api_key = None
305 if requires_api_key:
306 # Check the settings snapshot for the API key
307 api_key_path = f"search.engine.web.{engine_name}.api_key"
309 if settings_snapshot: 309 ↛ 320line 309 didn't jump to line 320 because the condition on line 309 was always true
310 api_key_setting = settings_snapshot.get(api_key_path)
312 if api_key_setting:
313 api_key = (
314 api_key_setting.get("value")
315 if isinstance(api_key_setting, dict)
316 else api_key_setting
317 )
319 # Still try to get from engine config if not found
320 if not api_key:
321 api_key = engine_config.get("api_key")
323 if not api_key:
324 logger.info(
325 f"Required API key for {engine_name} not found in settings."
326 )
327 return None
329 # Pass the API key in kwargs for engines that need it
330 if api_key: 330 ↛ 336line 330 didn't jump to line 336 because the condition on line 330 was always true
331 kwargs["api_key"] = api_key
333 # Warn about missing LLM but allow engine creation in degraded mode.
334 # All engines with requires_llm=True handle llm=None gracefully
335 # (e.g. skipping query optimization, using reliability-based sorting).
336 if engine_config.get("requires_llm", False) and not llm:
337 logger.warning(
338 f"Engine '{engine_name}' is configured with requires_llm=True but no LLM provided. "
339 f"Creating engine without LLM — some features (query optimization, relevance filtering) "
340 f"may be unavailable."
341 )
343 try:
344 # Load the engine class
345 module_path = engine_config["module_path"]
346 class_name = engine_config["class_name"]
348 engine_class = get_safe_module_class(module_path, class_name)
350 # Get the engine class's __init__ parameters to filter out unsupported ones
351 engine_init_signature = inspect.signature(engine_class.__init__)
352 engine_init_params = list(engine_init_signature.parameters.keys())
354 # Combine default parameters with provided ones
355 all_params = {**engine_config.get("default_params", {}), **kwargs}
357 # Filter out parameters that aren't accepted by the engine class
358 # Note: 'self' is always the first parameter of instance methods, so we skip it
359 filtered_params = {
360 k: v for k, v in all_params.items() if k in engine_init_params[1:]
361 }
363 # Always pass settings_snapshot if the engine accepts it
364 if "settings_snapshot" in engine_init_params[1:] and settings_snapshot:
365 filtered_params["settings_snapshot"] = settings_snapshot
367 # Pass programmatic_mode if the engine accepts it
368 if "programmatic_mode" in engine_init_params[1:]:
369 filtered_params["programmatic_mode"] = programmatic_mode
371 # Add LLM if required OR if provided and engine accepts it
372 if engine_config.get("requires_llm", False):
373 filtered_params["llm"] = llm
374 elif (
375 "llm" in engine_init_params[1:]
376 and llm
377 and "llm" not in filtered_params
378 ):
379 # If LLM was provided and engine accepts it, pass it through
380 filtered_params["llm"] = llm
381 logger.info(
382 f"Passing LLM to {engine_name} (engine accepts it and LLM was provided)"
383 )
385 # Add API key if required and not already in filtered_params
386 if ( 386 ↛ 391line 386 didn't jump to line 391 because the condition on line 386 was never true
387 engine_config.get("requires_api_key", False)
388 and "api_key" not in filtered_params
389 ):
390 # Use the api_key we got earlier from settings
391 if api_key:
392 filtered_params["api_key"] = api_key
394 logger.info(
395 f"Creating {engine_name} with filtered parameters: {filtered_params.keys()}"
396 )
398 # Create the engine instance with filtered parameters
399 engine = engine_class(**filtered_params)
401 # Stamp the registry name so BaseSearchEngine._verify_egress_scope()
402 # can check egress scope at run time.
403 if isinstance(engine, BaseSearchEngine):
404 engine._engine_name = engine_name
406 # Most engine subclasses do not name ``programmatic_mode`` in their
407 # signature (or accept it via **kwargs without forwarding to
408 # ``super().__init__``), so the constructor often falls back to the
409 # BaseSearchEngine default of False even when the API caller asked
410 # for True. Apply the requested mode post-construction so the
411 # engine's rate tracker matches.
412 if isinstance(engine, BaseSearchEngine) and (
413 engine.programmatic_mode != programmatic_mode
414 ):
415 engine._configure_programmatic_mode(programmatic_mode)
417 # Determine if this engine should use LLM relevance filtering
418 # Priority: per-engine setting > needs_llm_relevance_filter > global setting
419 #
420 # Rationale:
421 # - Engines with needs_llm_relevance_filter=True have poor native relevance ranking
422 # (keyword-only, no ML ranking) and benefit from LLM-based filtering
423 # - Well-ranked engines (Google, Brave) and semantic engines (Exa, Tavily)
424 # do not need this and should not waste LLM calls
425 # - The global skip_relevance_filter only affects unclassified engines
426 # - CrossEngineFilter still ranks combined results at the strategy level
427 should_filter = False
429 # Check for per-engine setting first (highest priority)
430 per_engine_key = f"search.engine.web.{engine_name}.default_params.enable_llm_relevance_filter"
431 if settings_snapshot and per_engine_key in settings_snapshot:
432 per_engine_setting = settings_snapshot[per_engine_key]
433 should_filter = (
434 per_engine_setting.get("value", False)
435 if isinstance(per_engine_setting, dict)
436 else per_engine_setting
437 )
438 logger.info(
439 f"Using per-engine setting for {engine_name}: "
440 f"enable_llm_relevance_filter={should_filter}"
441 )
442 else:
443 # Auto-detection based on engine attribute (medium priority)
444 if (
445 hasattr(engine_class, "needs_llm_relevance_filter")
446 and engine_class.needs_llm_relevance_filter
447 ):
448 should_filter = True
449 logger.info(
450 f"Auto-enabling LLM filtering for {engine_name} "
451 f"(needs_llm_relevance_filter=True)"
452 )
453 else:
454 # Global override only applies to engines without needs_llm_relevance_filter
455 if (
456 settings_snapshot
457 and "search.skip_relevance_filter" in settings_snapshot
458 ):
459 skip_filter_setting = settings_snapshot[
460 "search.skip_relevance_filter"
461 ]
462 skip_filter = (
463 skip_filter_setting.get("value", False)
464 if isinstance(skip_filter_setting, dict)
465 else skip_filter_setting
466 )
467 if skip_filter: 467 ↛ 475line 467 didn't jump to line 475 because the condition on line 467 was always true
468 should_filter = False
469 logger.debug(
470 f"Global skip_relevance_filter=True applied "
471 f"for {engine_name}"
472 )
474 # Apply the setting
475 if should_filter and hasattr(engine, "llm") and engine.llm:
476 engine.enable_llm_relevance_filter = True
477 logger.info(f"✓ Enabled LLM relevance filtering for {engine_name}")
478 elif should_filter:
479 logger.warning(
480 f"LLM relevance filtering requested for {engine_name} "
481 f"but no LLM is available — filtering skipped"
482 )
483 else:
484 logger.debug(f"LLM relevance filtering disabled for {engine_name}")
486 # Check if we need to wrap with full search capabilities
487 if kwargs.get("use_full_search", False) and engine_config.get(
488 "supports_full_search", False
489 ):
490 return _create_full_search_wrapper(
491 engine_name,
492 engine,
493 engine_config,
494 llm,
495 kwargs,
496 username,
497 settings_snapshot,
498 )
500 return engine # type: ignore[no-any-return]
502 except PolicyDeniedError:
503 # An engine __init__ (or a child engine / full-search wrapper it
504 # builds) may itself consult the PDP and raise. Re-raise so the
505 # denial surfaces to policy-aware callers and the audit trail,
506 # rather than being downgraded to a generic "failed to create"
507 # None — which would fail OPEN from an enforcement standpoint.
508 raise
509 except Exception as e:
510 safe_msg = scrub_error(e, api_key)
511 logger.exception(
512 f"Failed to create search engine '{engine_name}' ({type(e).__name__}): {safe_msg}"
513 )
514 return None
517def _create_full_search_wrapper(
518 engine_name: str,
519 base_engine: BaseSearchEngine,
520 engine_config: Dict[str, Any],
521 llm,
522 params: Dict[str, Any],
523 username: str | None = None,
524 settings_snapshot: Dict[str, Any] | None = None,
525) -> Optional[BaseSearchEngine]:
526 """Create a full search wrapper for the base engine if supported"""
527 try:
528 # Bound up here (re-populated below) so the except handler can
529 # always pull credential values out of it for scrub_error() —
530 # without the hoist, an exception before the dict is rebuilt would
531 # hit a NameError when trying to scrub the keys.
532 wrapper_params: Dict[str, Any] = {}
534 # Get full search class details from engine_config (already has
535 # registry-injected values from search_config()).
536 module_path = engine_config.get("full_search_module")
537 class_name = engine_config.get("full_search_class")
539 if not module_path or not class_name:
540 logger.warning(
541 f"Full search configuration missing for {engine_name}"
542 )
543 return base_engine
545 # Import the full search class
546 full_search_class = get_safe_module_class(module_path, class_name)
548 # Get the wrapper's __init__ parameters to filter out unsupported ones
549 wrapper_init_signature = inspect.signature(full_search_class.__init__)
550 wrapper_init_params = list(wrapper_init_signature.parameters.keys())[
551 1:
552 ] # Skip 'self'
554 # Extract relevant parameters for the full search wrapper
555 wrapper_params = {
556 k: v for k, v in params.items() if k in wrapper_init_params
557 }
559 # Special case for SerpAPI which needs the API key directly
560 if (
561 engine_name == "serpapi"
562 and "serpapi_api_key" in wrapper_init_params
563 ):
564 # Check settings snapshot for API key
565 serpapi_api_key = None
566 if settings_snapshot: 566 ↛ 576line 566 didn't jump to line 576 because the condition on line 566 was always true
567 serpapi_setting = settings_snapshot.get(
568 "search.engine.web.serpapi.api_key"
569 )
570 if serpapi_setting: 570 ↛ 576line 570 didn't jump to line 576 because the condition on line 570 was always true
571 serpapi_api_key = (
572 serpapi_setting.get("value")
573 if isinstance(serpapi_setting, dict)
574 else serpapi_setting
575 )
576 if serpapi_api_key: 576 ↛ 580line 576 didn't jump to line 580 because the condition on line 576 was always true
577 wrapper_params["serpapi_api_key"] = serpapi_api_key
579 # Map some parameter names to what the wrapper expects
580 if (
581 "language" in params
582 and "search_language" not in params
583 and "language" in wrapper_init_params
584 ):
585 wrapper_params["language"] = params["language"]
587 if (
588 "safesearch" not in wrapper_params
589 and "safe_search" in params
590 and "safesearch" in wrapper_init_params
591 ):
592 wrapper_params["safesearch"] = (
593 "active" if params["safe_search"] else "off"
594 )
596 # Special case for Brave which needs the API key directly
597 if engine_name == "brave" and "api_key" in wrapper_init_params:
598 # Check settings snapshot for API key
599 brave_api_key = None
600 if settings_snapshot:
601 brave_setting = settings_snapshot.get(
602 "search.engine.web.brave.api_key"
603 )
604 if brave_setting: 604 ↛ 611line 604 didn't jump to line 611 because the condition on line 604 was always true
605 brave_api_key = (
606 brave_setting.get("value")
607 if isinstance(brave_setting, dict)
608 else brave_setting
609 )
611 if brave_api_key:
612 wrapper_params["api_key"] = brave_api_key
614 # Map some parameter names to what the wrapper expects
615 if (
616 "language" in params
617 and "search_language" not in params
618 and "language" in wrapper_init_params
619 ):
620 wrapper_params["language"] = params["language"]
622 if (
623 "safesearch" not in wrapper_params
624 and "safe_search" in params
625 and "safesearch" in wrapper_init_params
626 ):
627 wrapper_params["safesearch"] = (
628 "moderate" if params["safe_search"] else "off"
629 )
631 # Always include llm if it's a parameter
632 if "llm" in wrapper_init_params:
633 wrapper_params["llm"] = llm
635 # If the wrapper needs the base engine and has a parameter for it
636 if "web_search" in wrapper_init_params:
637 wrapper_params["web_search"] = base_engine
639 logger.debug(
640 f"Creating full search wrapper for {engine_name} with filtered parameters: {wrapper_params.keys()}"
641 )
643 # Create the full search wrapper with filtered parameters
644 service: BaseSearchEngine = full_search_class(**wrapper_params)
645 return service
647 except Exception as e:
648 # Extract any credential values the wrapper was going to receive
649 # so a constructor exception echoing them gets literal-redacted
650 # (scrub_error skips None/falsy values and str-coerces the rest).
651 safe_msg = scrub_error(
652 e,
653 wrapper_params.get("api_key"),
654 wrapper_params.get("serpapi_api_key"),
655 )
656 logger.exception(
657 f"Failed to create full search wrapper for {engine_name} ({type(e).__name__}): {safe_msg}"
658 )
659 return base_engine
662def get_search(
663 search_tool: str,
664 llm_instance,
665 max_results: int = 10,
666 region: str = "us",
667 time_period: str = "all",
668 safe_search: bool = True,
669 search_snippets_only: bool = False,
670 search_language: str = "English",
671 max_filtered_results: Optional[int] = None,
672 settings_snapshot: Dict[str, Any] | None = None,
673 programmatic_mode: bool = False,
674):
675 """
676 Get search tool instance based on the provided parameters.
678 Args:
679 search_tool: Name of the search engine to use
680 llm_instance: Language model instance
681 max_results: Maximum number of search results
682 region: Search region/locale
683 time_period: Time period for search results
684 safe_search: Whether to enable safe search
685 search_snippets_only: Whether to return just snippets (vs. full content)
686 search_language: Language for search results
687 max_filtered_results: Maximum number of results to keep after filtering
688 programmatic_mode: If True, disables database operations and metrics tracking
690 Returns:
691 Initialized search engine instance
692 """
693 # Common parameters
694 params = {
695 "max_results": max_results,
696 "llm": llm_instance, # Only used by engines that need it
697 }
699 # Add max_filtered_results if provided
700 if max_filtered_results is not None:
701 params["max_filtered_results"] = max_filtered_results
703 # Add engine-specific parameters
704 if search_tool in [
705 "duckduckgo",
706 "serpapi",
707 "google_pse",
708 "brave",
709 "mojeek",
710 ]:
711 params.update(
712 {
713 "region": region,
714 "safe_search": safe_search,
715 "use_full_search": not search_snippets_only,
716 }
717 )
719 if search_tool == "searxng":
720 params["use_full_search"] = not search_snippets_only
722 if search_tool in ["serpapi", "brave", "google_pse", "wikinews"]:
723 params["search_language"] = search_language
725 if search_tool == "tinyfish": 725 ↛ 726line 725 didn't jump to line 726 because the condition on line 725 was never true
726 params["location"] = region.upper()
727 params["language"] = search_language
728 params["search_snippets_only"] = search_snippets_only
730 if search_tool == "sofya":
731 # Sofya derives its request depth from this: the paid "basic" depth
732 # (inline page content) is only requested when full content is used.
733 params["search_snippets_only"] = search_snippets_only
735 if search_tool == "wikinews":
736 params["search_snippets_only"] = search_snippets_only
737 params["adaptive_search"] = bool(
738 (settings_snapshot or {})
739 .get("search.engine.web.wikinews.adaptive_search", {})
740 .get("value", True)
741 )
743 if search_tool in TIME_PERIOD_ENGINES:
744 params["time_period"] = time_period
746 # Create and return the search engine
747 logger.info(
748 f"Creating search engine for tool: {search_tool} (type: {type(search_tool)}) with params: {params.keys()}"
749 )
750 logger.info(
751 f"About to call create_search_engine with search_tool={search_tool}, settings_snapshot type={type(settings_snapshot)}"
752 )
753 logger.info(
754 f"Params being passed to create_search_engine: {list(params.keys()) if isinstance(params, dict) else type(params)}"
755 )
757 engine = create_search_engine(
758 search_tool,
759 settings_snapshot=settings_snapshot,
760 programmatic_mode=programmatic_mode,
761 **params,
762 )
764 # Add debugging to check if engine is None
765 if engine is None:
766 logger.error(
767 f"Failed to create search engine for {search_tool} - returned None"
768 )
769 else:
770 engine_type = type(engine).__name__
771 logger.info(
772 f"Successfully created search engine of type: {engine_type}"
773 )
774 # Check if the engine has run method
775 if hasattr(engine, "run"): 775 ↛ 778line 775 didn't jump to line 778 because the condition on line 775 was always true
776 logger.info(f"Engine has 'run' method: {engine.run}")
777 else:
778 logger.error("Engine does NOT have 'run' method!")
780 # Check engine availability flag / method
781 avail = None
782 if getattr(engine, "_is_available", None) is not None:
783 avail = getattr(engine, "_is_available")
784 elif hasattr(engine, "is_available"):
785 if callable(engine.is_available): 785 ↛ 808line 785 didn't jump to line 808 because the condition on line 785 was always true
786 try:
787 avail = engine.is_available(
788 settings_snapshot=settings_snapshot
789 )
790 except TypeError:
791 try:
792 avail = engine.is_available()
793 except Exception as exc:
794 logger.debug(
795 "Engine availability fallback failed for {} ({})",
796 engine_type,
797 type(exc).__name__,
798 )
799 avail = None
800 except Exception as exc:
801 logger.debug(
802 "Engine availability check failed for {} ({})",
803 engine_type,
804 type(exc).__name__,
805 )
806 avail = None
807 else:
808 avail = engine.is_available
809 if avail is not None:
810 logger.info(f"Engine availability flag: {avail}")
812 return engine