Coverage for src/local_deep_research/web_search_engines/engines/search_engine_searxng.py: 96%
271 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1import enum
2import json
3from typing import Any, Dict, List, Optional
5import requests
6from langchain_core.language_models import BaseLLM
8from ...security import redact_url_for_log
9from ...security.egress.policy import PolicyDeniedError
10from ...security.safe_requests import safe_get
12from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
13from ...security.secure_logging import logger
14from ._searxng_rate_limiter import respect_rate_limit
15from ..engine_availability import SearchFailure
17# The operator-approval resolver lives in security/egress/validators.py next
18# to the allowlist matcher it consults (dependency direction: engine →
19# security, same as everything else here). Re-exported under the historical
20# private name so this module's call sites and the existing test imports
21# keep their import path (the signature itself gained the instance-URL
22# argument when the resolver was generalized).
23from ...security.egress.validators import (
24 resolve_searxng_allow_private_ips as _resolve_searxng_allow_private_ips,
25 resolve_searxng_allow_private_result_fetch,
26)
29@enum.unique
30class SafeSearchSetting(enum.IntEnum):
31 """
32 Acceptable settings for safe search.
33 """
35 OFF = 0
36 MODERATE = 1
37 STRICT = 2
40class SearXNGSearchEngine(BaseSearchEngine):
41 """
42 SearXNG search engine implementation that requires an instance URL provided via
43 environment variable or configuration. Designed for ethical usage with proper
44 rate limiting and single-instance approach.
45 """
47 # Mark as public search engine
48 is_public = True
49 egress_sensitivity = Sensitivity.NON_SENSITIVE
50 egress_exposure = Exposure.EXPOSING
51 # Mark as generic search engine (general web search)
52 is_generic = True
53 # The egress engine-selection gate uses the static is_public flag above —
54 # SearXNG always queries the internet regardless of where it's hosted, so
55 # a localhost instance_url does NOT reclassify it as private (the PDP's
56 # URL override is fail-up only and never relaxes a public nature).
57 url_setting = "search.engine.web.searxng.default_params.instance_url"
59 @staticmethod
60 def _normalize_list(value, *, setting_name: str = "list setting"):
61 """Ensure *value* is a ``list[str]`` or ``None``.
63 Settings saved via the web UI may arrive as raw JSON strings
64 (e.g. ``'[\\r\\n "general"\\r\\n]'``) instead of parsed lists.
65 This helper decodes such strings so that ``",".join()`` later
66 works on list items rather than individual characters (issue #1030).
68 Input that is neither valid JSON nor a plain comma-separated list
69 still produces a list, because the comma fallback splits whatever it
70 is given. That is right for ``kagi,brave`` and wrong for
71 ``['kagi', 'brave']``, which yields ``["['kagi'", "'brave']"]`` --
72 names no engine matches, so the filter silently selects nothing while
73 looking configured. Warn when the input was evidently meant to be
74 JSON, so the failure is visible rather than inferred from empty
75 results.
76 """
77 if value is None:
78 return None
79 if isinstance(value, list):
80 return value
81 if isinstance(value, str):
82 stripped = value.strip()
83 if stripped:
84 try:
85 parsed = json.loads(stripped)
86 if isinstance(parsed, list):
87 return [str(item) for item in parsed]
88 except (json.JSONDecodeError, ValueError, RecursionError):
89 # Only a bracketed/braced value was *trying* to be JSON.
90 # Genuine comma-separated input takes this path too and
91 # is not a mistake, so it must not warn.
92 if stripped[0] in "[{":
93 logger.warning(
94 "{} is not valid JSON ({!r}); falling back to "
95 "comma-splitting, which keeps brackets and quotes "
96 "as part of each entry. Use double-quoted JSON, "
97 'e.g. ["kagi", "brave"].',
98 setting_name,
99 stripped[:100],
100 )
101 # Comma-separated fallback
102 return [
103 item.strip() for item in stripped.split(",") if item.strip()
104 ]
105 return None
107 def _is_valid_search_result(self, url: str) -> bool:
108 """
109 Check if a parsed result is a valid search result vs an error page.
111 When SearXNG's backend engines fail or get rate-limited, it returns
112 error/stats pages that shouldn't be treated as search results.
114 Returns False for:
115 - Relative URLs (don't start with http:// or https://, case-insensitive)
116 - URLs pointing to the SearXNG instance itself (catches /stats, /preferences, etc.)
117 """
118 # Must have an absolute URL (case-insensitive scheme check)
119 if not url or not url.lower().startswith(("http://", "https://")):
120 return False
122 # Reject URLs pointing back to the SearXNG instance itself
123 # This catches all internal pages like /stats?engine=, /preferences, /about
124 if url.startswith(self.instance_url):
125 return False
127 return True
129 def _private_url_blocked_by_gate(self) -> bool:
130 """Whether the instance URL was refused *only* because it is
131 private/loopback/link-local with no operator approval active
132 (no allowlist match, no blanket opt-in, no env-locked URL).
134 Used to decide whether to surface the operator-remediation hint
135 (allowlist / blanket opt-in / env-lock). Detects
136 the case precisely so the hint fires for the common localhost
137 self-host misconfiguration but NOT for unrelated failures (DNS /
138 timeout / a genuinely-public URL) nor for an ALWAYS-blocked
139 cloud-metadata address — which the operator opt-in would never lift,
140 so pointing at that flag would be misleading.
142 Returns True when the URL is blocked under the current (gate-OFF)
143 posture yet WOULD pass with private egress allowed: that difference
144 is exactly the private-URL gate, and the opt-in is the fix. Fails
145 CLOSED (False) on any error so hint detection can never break init.
146 """
147 if self._allow_private_ips:
148 return False
149 try:
150 from ...security.ssrf_validator import validate_url
152 return not validate_url(
153 self.instance_url, allow_private_ips=False
154 ) and validate_url(self.instance_url, allow_private_ips=True)
155 except Exception: # noqa: BLE001 - hint detection is best-effort
156 return False
158 def __init__(
159 self,
160 max_results: int = 15,
161 instance_url: str = "http://localhost:8080",
162 categories: Optional[List[str]] = None,
163 engines: Optional[List[str]] = None,
164 language: str = "en",
165 safe_search: str = SafeSearchSetting.OFF.name,
166 time_range: Optional[str] = None,
167 result_format: str = "html",
168 delay_between_requests: float = 0.0,
169 llm: Optional[BaseLLM] = None,
170 max_filtered_results: Optional[int] = None,
171 include_full_content: bool = True,
172 settings_snapshot: Optional[Dict[str, Any]] = None,
173 search_snippets_only: bool = True,
174 **kwargs,
175 ): # API key is actually the instance URL
176 """
177 Initialize the SearXNG search engine with ethical usage patterns.
179 Args:
180 max_results: Maximum number of search results
181 instance_url: URL of your SearXNG instance (preferably self-hosted)
182 categories: List of SearXNG categories to search in (general, images, videos, news, etc.)
183 engines: List of engines to use (google, bing, duckduckgo, etc.)
184 language: Language code for search results
185 safe_search: Safe search level (0=off, 1=moderate, 2=strict)
186 time_range: Time range for results (day, week, month, year)
187 result_format: Response format to request from SearXNG, either
188 "html" (default, scraped) or "json". Use "json" for instances
189 whose ``search.formats`` enables JSON but not HTML.
190 delay_between_requests: Seconds to wait between requests
191 llm: Language model for relevance filtering
192 max_filtered_results: Maximum number of results to keep after filtering
193 include_full_content: Retained for backward compatibility; full-content
194 retrieval is handled via the factory wrapper.
195 search_snippets_only: Whether to return only search snippets
196 """
198 # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
199 super().__init__(
200 llm=llm,
201 max_filtered_results=max_filtered_results,
202 max_results=max_results,
203 include_full_content=include_full_content,
204 settings_snapshot=settings_snapshot,
205 search_snippets_only=search_snippets_only,
206 **kwargs,
207 )
209 # Validate and normalize the instance URL if provided
210 self.instance_url = instance_url.rstrip("/")
211 logger.info(
212 f"SearXNG initialized with instance URL: {redact_url_for_log(self.instance_url)}"
213 )
214 # Whether this instance's private/loopback URL may be reached. Derived
215 # from the operator opt-in (``LDR_SEARCH_ALLOW_PRIVATE_ENGINE_URLS``)
216 # or an env-locked instance URL only — NOT from the user-editable
217 # egress scope — rather than hard-coded True, so under the default
218 # PUBLIC_ONLY posture a user-editable instance_url pointed at an
219 # internal host is NOT fetched. Cloud-metadata IPs remain blocked
220 # either way. See ``_resolve_searxng_allow_private_ips`` for the rules.
221 self._allow_private_ips = _resolve_searxng_allow_private_ips(
222 self.instance_url
223 )
224 # Whether full-content fetches may follow this instance's RESULT
225 # URLs into private/loopback hosts (e.g. an internal wiki it
226 # indexes). Deliberately a separate, narrower grant than the
227 # instance gate above: that one is satisfied by every env-locked
228 # (Docker) deployment, and a public engine proxies the public web,
229 # so it must not by itself license fetching whatever private URLs
230 # the index returns. Requires the instance gate AND the env-only
231 # ``LDR_SEARCH_ALLOW_PRIVATE_RESULT_FETCH`` opt-in; see
232 # ``resolve_engine_allow_private_result_fetch``. Overrides the
233 # ``BaseSearchEngine`` default (False); the factory hands it to the
234 # full-search wrapper it builds around this engine (issue #2477).
235 self.allow_private_result_fetch = (
236 resolve_searxng_allow_private_result_fetch(self.instance_url)
237 )
238 try:
239 # Make sure it's accessible.
240 response = safe_get(
241 self.instance_url,
242 timeout=5,
243 allow_private_ips=self._allow_private_ips,
244 )
245 if response.status_code == 200:
246 logger.info("SearXNG instance is accessible.")
247 self._is_available = True
248 else:
249 self._is_available = False
250 self._record_search_failure(
251 requests.HTTPError(response=response)
252 )
253 logger.error(
254 f"Failed to access SearXNG instance at {redact_url_for_log(self.instance_url)}. Status code: {response.status_code}"
255 )
256 except (requests.RequestException, ValueError) as e:
257 self._is_available = False
258 self._record_search_failure(e)
259 safe_msg = self._scrub_error(e)
260 logger.exception(
261 f"Error while trying to access SearXNG instance at {redact_url_for_log(self.instance_url)} ({type(e).__name__}): {safe_msg}"
262 )
263 private_url_hint = self._private_url_blocked_by_gate()
264 else:
265 private_url_hint = False
266 if private_url_hint:
267 self.last_search_failure = SearchFailure(
268 "Instance URL is private/loopback and has no server approval; "
269 "see docs/SearXNG-Setup.md",
270 None,
271 )
272 # The dominant self-host case: default localhost:8080 SearXNG
273 # with no operator opt-in. The generic exception above never
274 # names the fix, so surface it explicitly on this runtime
275 # path — at ERROR, so the actionable message is at least as
276 # loud as the generic one preceding it. (Emitted outside the
277 # handler: there is no second traceback to attach, and the
278 # logger.exception-in-handler rule stays satisfied.)
279 logger.error(
280 "SearXNG engine disabled: instance URL "
281 f"{redact_url_for_log(self.instance_url)} is a private / "
282 "loopback / link-local address, which is not fetched by "
283 "default. To use a self-hosted SearXNG on localhost/LAN, "
284 "the server operator must approve it in the server "
285 "environment and restart: add the URL origin to "
286 "LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST, or env-lock "
287 "the URL via "
288 "LDR_SEARCH_ENGINE_WEB_SEARXNG_DEFAULT_PARAMS_INSTANCE_URL "
289 "(as the bundled docker-compose does), or set "
290 "LDR_SEARCH_ALLOW_PRIVATE_ENGINE_URLS=true. Only one of "
291 "these is needed. Cloud-metadata addresses stay blocked "
292 "regardless. See docs/SearXNG-Setup.md."
293 )
295 # Add debug logging for all parameters
296 logger.info(
297 f"SearXNG init params: max_results={max_results}, language={language}, "
298 f"max_filtered_results={max_filtered_results}, is_available={self._is_available}"
299 )
301 self.max_results = max_results
302 self.categories = self._normalize_list(
303 categories, setting_name="SearXNG categories"
304 ) or ["general"]
305 self.engines = self._normalize_list(
306 engines, setting_name="SearXNG engines"
307 )
308 self.language = language
309 try:
310 # Handle both string names and integer values
311 if isinstance(safe_search, int) or (
312 isinstance(safe_search, str) and str(safe_search).isdigit()
313 ):
314 self.safe_search = SafeSearchSetting(int(safe_search))
315 else:
316 self.safe_search = SafeSearchSetting[safe_search]
317 except (ValueError, KeyError) as e:
318 safe_msg = self._scrub_error(e)
319 logger.exception(
320 f"'{safe_search}' is not a valid safe search setting ({type(e).__name__}). Disabling safe search: {safe_msg}"
321 )
322 self.safe_search = SafeSearchSetting.OFF
323 self.time_range = time_range
325 normalized_format = str(result_format).strip().lower()
326 if normalized_format not in ("html", "json"):
327 logger.warning(
328 f"'{result_format}' is not a supported SearXNG result_format "
329 "('html' or 'json'). Falling back to 'html'."
330 )
331 normalized_format = "html"
332 self.result_format = normalized_format
334 self.delay_between_requests = float(delay_between_requests)
336 if self._is_available:
337 self.search_url = f"{self.instance_url}/search"
338 logger.info(
339 f"SearXNG engine initialized with instance: {redact_url_for_log(self.instance_url)}"
340 )
341 logger.info(
342 f"Rate limiting set to {self.delay_between_requests} seconds between requests"
343 )
345 # Full-content retrieval is the factory wrapper's job — see
346 # ``search_engine_factory._create_full_search_wrapper``. The
347 # wrapper fetches pages via ``batch_fetch_and_extract`` and
348 # populates ``full_content`` on each result; the inner engine
349 # only emits snippets from ``_get_previews``.
350 if self.allow_private_result_fetch:
351 # The grant is an allow, not a deny, so leave an operator-
352 # visible trace of it (the closed-gate and policy-denial
353 # paths already log). The factory reads the attribute when
354 # it builds that wrapper (issue #2477).
355 # The wrapper applies the grant only under the unprotected
356 # egress scope (see ``FullSearchResults.
357 # _effective_allow_private_ips``).
358 logger.info(
359 "SearXNG private result-fetch grant "
360 "(LDR_SEARCH_ALLOW_PRIVATE_RESULT_FETCH) is set for "
361 f"instance: {redact_url_for_log(self.instance_url)}; "
362 "full-content fetches may reach private result URLs "
363 "only under the unprotected egress scope"
364 )
366 def _respect_rate_limit(self):
367 """Apply self-imposed rate limiting between requests.
369 Rate-limit state is shared across all `SearXNGSearchEngine`
370 instances targeting the same ``instance_url`` (see
371 ``_searxng_rate_limiter.py``) so the configured delay applies
372 even when the research agent constructs a fresh engine for
373 every tool call.
374 """
375 respect_rate_limit(self.instance_url, self.delay_between_requests)
377 def _get_search_results(
378 self, query: str, max_results: Optional[int] = None
379 ) -> List[Dict[str, Any]]:
380 """
381 Get search results from SearXNG with ethical rate limiting.
383 Args:
384 query: The search query
385 max_results: Optional per-request result limit. Uses the engine's
386 configured limit when omitted.
388 Returns:
389 List of search results from SearXNG
390 """
391 raw_limit = self.max_results if max_results is None else max_results
392 result_limit = max(0, raw_limit)
394 if not self._is_available:
395 logger.error(
396 "SearXNG engine failed its availability check - cannot run search"
397 )
398 return []
400 logger.info("SearXNG running search")
402 try:
403 self._respect_rate_limit()
405 initial_headers = {
406 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
407 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
408 "Accept-Language": "en-US,en;q=0.9",
409 }
411 try:
412 initial_response = safe_get(
413 self.instance_url,
414 headers=initial_headers,
415 timeout=10,
416 allow_private_ips=self._allow_private_ips,
417 )
418 cookies = initial_response.cookies
419 except Exception as e:
420 safe_msg = self._scrub_error(e)
421 logger.exception(
422 f"Failed to get initial cookies ({type(e).__name__}): {safe_msg}"
423 )
424 cookies = None
426 params = {
427 "q": query,
428 "categories": ",".join(self.categories),
429 "language": self.language,
430 "format": self.result_format,
431 "pageno": 1,
432 "safesearch": self.safe_search.value,
433 "count": result_limit,
434 }
436 if self.engines:
437 params["engines"] = ",".join(self.engines)
439 if self.time_range: 439 ↛ 440line 439 didn't jump to line 440 because the condition on line 439 was never true
440 params["time_range"] = self.time_range
442 # Browser-like headers
443 headers = {
444 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
445 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
446 "Accept-Language": "en-US,en;q=0.9",
447 "Referer": self.instance_url + "/",
448 "Connection": "keep-alive",
449 "Upgrade-Insecure-Requests": "1",
450 }
452 logger.info(
453 f"Sending request to SearXNG instance at {redact_url_for_log(self.instance_url)}"
454 )
455 response = safe_get(
456 self.search_url,
457 params=params,
458 headers=headers,
459 cookies=cookies,
460 timeout=15,
461 allow_private_ips=self._allow_private_ips,
462 )
464 if response.status_code == 200:
465 if self.result_format == "json":
466 return self._parse_json_results(
467 response, query, result_limit
468 )
470 try:
471 from bs4 import BeautifulSoup
473 soup = BeautifulSoup(response.text, "html.parser")
474 results: List[Dict[str, Any]] = []
476 result_elements = soup.select(".result-item")
478 if not result_elements:
479 result_elements = soup.select(".result")
481 if not result_elements:
482 result_elements = soup.select("article")
484 if not result_elements:
485 logger.debug(
486 f"Classes found in HTML: {[c['class'] for c in soup.select('[class]') if 'class' in c.attrs][:10]}"
487 )
488 result_elements = soup.select('div[id^="result"]')
490 logger.info(
491 f"Found {len(result_elements)} search result elements"
492 )
494 for idx, result_element in enumerate(result_elements):
495 if len(results) >= result_limit:
496 break
498 title_element = (
499 result_element.select_one(".result-title")
500 or result_element.select_one(".title")
501 or result_element.select_one("h3")
502 or result_element.select_one("a[href]")
503 )
505 url_element = (
506 result_element.select_one(".result-url")
507 or result_element.select_one(".url")
508 or result_element.select_one("a[href]")
509 )
511 content_element = (
512 result_element.select_one(".result-content")
513 or result_element.select_one(".content")
514 or result_element.select_one(".snippet")
515 or result_element.select_one("p")
516 )
518 title = (
519 title_element.get_text(" ", strip=True)
520 if title_element
521 else ""
522 )
524 url = ""
525 if url_element and url_element.has_attr("href"):
526 url = self._clean_result_url(url_element["href"])
527 elif url_element: 527 ↛ 530line 527 didn't jump to line 530 because the condition on line 527 was always true
528 url = url_element.get_text(strip=True)
530 content = (
531 content_element.get_text(" ", strip=True)
532 if content_element
533 else ""
534 )
536 if (
537 not url
538 and title_element
539 and title_element.has_attr("href")
540 ):
541 url = self._clean_result_url(title_element["href"])
543 logger.debug(
544 f"Extracted result {idx}: title={title[:30]}..., url={redact_url_for_log(url)}, content={content[:30]}..."
545 )
547 # Add to results only if it's a valid search result
548 # (not an error page or internal SearXNG page)
549 if self._is_valid_search_result(url):
550 results.append(
551 {
552 "title": title,
553 "url": url,
554 "content": content,
555 "engine": "searxng",
556 "category": "general",
557 }
558 )
559 else:
560 # Check if this is a backend engine failure
561 if url and "/stats?engine=" in url: 561 ↛ 571line 561 didn't jump to line 571 because the condition on line 561 was always true
562 try:
563 engine_name = url.split("/stats?engine=")[
564 1
565 ].split("&")[0]
566 logger.warning(
567 f"SearXNG backend engine failed or rate-limited: {engine_name}"
568 )
569 except (IndexError, AttributeError):
570 pass # Couldn't parse engine name
571 logger.debug(
572 f"Filtered invalid SearXNG result: title={title!r}, url={redact_url_for_log(url)}"
573 )
575 if results:
576 logger.info(
577 f"SearXNG returned {len(results)} valid results from HTML parsing"
578 )
579 else:
580 # Not a search failure: SearXNG routinely lists
581 # suspended backends, and neither the page nor the
582 # JSON API says which backends were queried, so an
583 # empty page cannot prove every backend failed.
584 logger.warning(
585 "SearXNG returned no valid results. "
586 "This may indicate SearXNG backend engine issues or rate limiting."
587 )
588 return results
590 except ImportError as e:
591 self.last_search_failure = SearchFailure(
592 "HTML parser is unavailable; check server dependencies",
593 None,
594 )
595 safe_msg = self._scrub_error(e)
596 logger.exception(
597 f"BeautifulSoup not available for HTML parsing ({type(e).__name__}): {safe_msg}"
598 )
599 return []
600 except Exception as e:
601 self._record_search_failure(e)
602 safe_msg = self._scrub_error(e)
603 logger.exception(
604 f"Error parsing HTML results ({type(e).__name__}): {safe_msg}"
605 )
606 return []
607 else:
608 self._record_search_failure(
609 requests.HTTPError(response=response)
610 )
611 logger.error(
612 f"SearXNG returned status code {response.status_code}"
613 )
614 return []
616 except Exception as e:
617 self._record_search_failure(e)
618 safe_msg = self._scrub_error(e)
619 logger.exception(
620 f"Error getting SearXNG results ({type(e).__name__}): {safe_msg}"
621 )
622 return []
624 def _parse_json_results(
625 self, response: Any, query: str, max_results: Optional[int] = None
626 ) -> List[Dict[str, Any]]:
627 """Parse SearXNG's ``format=json`` response.
629 SearXNG returns a JSON document whose ``results`` array already
630 carries structured ``url``/``title``/``content`` fields, so no HTML
631 scraping is needed. The output shape matches the HTML parse path
632 (``title``, ``url``, ``content``, ``engine``, ``category``) and the
633 same ``_is_valid_search_result`` filter is applied so both paths
634 reject SearXNG's own internal/stats pages identically.
636 Args:
637 response: HTTP response object containing JSON payload
638 query: The search query string
639 max_results: Optional per-request result limit. Uses the engine's
640 configured limit when omitted.
642 Returns:
643 List of search results from SearXNG
644 """
645 raw_limit = self.max_results if max_results is None else max_results
646 result_limit = max(0, raw_limit)
648 try:
649 data = response.json()
650 except (json.JSONDecodeError, ValueError) as e:
651 self.last_search_failure = SearchFailure(
652 "Invalid JSON response; check the SearXNG response format"
653 )
654 safe_msg = self._scrub_error(e)
655 logger.exception(
656 f"Error parsing JSON results ({type(e).__name__}): {safe_msg}. "
657 "Ensure the SearXNG instance enables the 'json' format."
658 )
659 return []
661 # Surface backend engine failures the same way the HTML path does.
662 unresponsive = data.get("unresponsive_engines") or []
663 if unresponsive:
664 logger.warning(
665 f"SearXNG reported unresponsive engines: {unresponsive}"
666 )
668 results: List[Dict[str, Any]] = []
669 for item in data.get("results", []):
670 if len(results) >= result_limit:
671 break
672 if not isinstance(item, dict): 672 ↛ 673line 672 didn't jump to line 673 because the condition on line 672 was never true
673 continue
675 url = self._clean_result_url(item.get("url"))
676 if not self._is_valid_search_result(url):
677 logger.debug(
678 f"Filtered invalid SearXNG result: url={redact_url_for_log(url)}"
679 )
680 continue
682 results.append(
683 {
684 "title": item.get("title") or "",
685 "url": url,
686 "content": item.get("content") or "",
687 "engine": item.get("engine") or "searxng",
688 "category": item.get("category") or "general",
689 }
690 )
692 if results:
693 logger.info(
694 f"SearXNG returned {len(results)} valid results from JSON parsing"
695 )
696 else:
697 # ``unresponsive_engines`` lists only the backends that failed,
698 # not the set that was queried, so this cannot show that every
699 # backend failed. Treat it as no matches (see the HTML path).
700 logger.warning(
701 "SearXNG returned no valid results. "
702 "This may indicate SearXNG backend engine issues or rate limiting."
703 )
704 return results
706 def _get_previews(self, query: str) -> List[Dict[str, Any]]:
707 """
708 Get preview information for SearXNG search results.
710 Args:
711 query: The search query
713 Returns:
714 List of preview dictionaries
715 """
716 if not self._is_available:
717 logger.warning("SearXNG engine failed its availability check")
718 return []
720 logger.info("Getting SearXNG previews")
722 results = self._get_search_results(query)
724 if not results:
725 logger.warning("No SearXNG results found")
726 return []
728 previews = []
729 for i, result in enumerate(results):
730 title = result.get("title", "")
731 url = self._clean_result_url(result.get("url"))
732 content = result.get("content", "")
734 preview = {
735 "id": url or f"searxng-result-{i}",
736 "title": title,
737 "link": url,
738 "snippet": content,
739 "engine": result.get("engine", ""),
740 "category": result.get("category", ""),
741 }
743 previews.append(preview)
745 return previews
747 def _get_full_content(
748 self, relevant_items: List[Dict[str, Any]]
749 ) -> List[Dict[str, Any]]:
750 """Return items unchanged.
752 Full-content retrieval is the factory wrapper's job. The wrapper
753 fetches pages via ``batch_fetch_and_extract`` and populates
754 ``full_content`` on each result; the inner engine never fetches
755 pages itself.
756 """
757 return relevant_items
759 def invoke(self, query: str) -> List[Dict[str, Any]]:
760 """Compatibility method for LangChain tools"""
761 return self.run(query)
763 def results(
764 self, query: str, max_results: Optional[int] = None
765 ) -> List[Dict[str, Any]]:
766 """
767 Get search results in a format compatible with other search engines.
769 Args:
770 query: The search query
771 max_results: Optional override for maximum results
773 Returns:
774 List of search result dictionaries
775 """
776 if not self._is_available:
777 return []
779 results = self._get_search_results(query, max_results=max_results)
781 formatted_results = []
782 for result in results:
783 formatted_results.append(
784 {
785 "title": result.get("title", ""),
786 "link": self._clean_result_url(result.get("url")),
787 "snippet": result.get("content", ""),
788 }
789 )
791 return formatted_results
793 @staticmethod
794 def get_self_hosting_instructions() -> str:
795 """
796 Get instructions for self-hosting a SearXNG instance.
798 Returns:
799 String with installation instructions
800 """
801 return """
802# SearXNG Self-Hosting Instructions
804The most ethical way to use SearXNG is to host your own instance. Here's how:
806## Using Docker (easiest method)
8081. Install Docker if you don't have it already
8092. Run these commands:
811```bash
812# Pull the SearXNG Docker image
813docker pull searxng/searxng
815# Run SearXNG (will be available at http://localhost:8080)
816docker run -d -p 8080:8080 --name searxng searxng/searxng
817```
819## Using Docker Compose (recommended for production)
8211. Create a file named `docker-compose.yml` with the following content:
823```yaml
824version: '3'
825services:
826 searxng:
827 container_name: searxng
828 image: searxng/searxng
829 ports:
830 - "8080:8080"
831 volumes:
832 - ./searxng:/etc/searxng
833 environment:
834 - SEARXNG_BASE_URL=http://localhost:8080/
835 restart: unless-stopped
836```
8382. Run with Docker Compose:
840```bash
841docker-compose up -d
842```
844For more detailed instructions and configuration options, visit:
845https://searxng.github.io/searxng/admin/installation.html
846"""
848 def run(
849 self, query: str, research_context: Dict[str, Any] | None = None
850 ) -> List[Dict[str, Any]]:
851 """
852 Override BaseSearchEngine run method to add SearXNG-specific error handling.
853 """
854 if not self._is_available:
855 logger.error(
856 "SearXNG run method called but engine failed its availability check"
857 )
858 return []
860 logger.info(
861 f"SearXNG instance URL: {redact_url_for_log(self.instance_url)}"
862 )
864 try:
865 # Call the parent class's run method
866 results = super().run(query, research_context=research_context)
867 logger.info(f"SearXNG search completed with {len(results)} results")
868 return results
869 except PolicyDeniedError:
870 raise
871 except Exception as e:
872 safe_msg = self._scrub_error(e)
873 logger.exception(
874 f"Error in SearXNG run method ({type(e).__name__}): {safe_msg}"
875 )
876 # Return empty results on error
877 return []