Coverage for src/local_deep_research/web_search_engines/engines/search_engine_serply.py: 95%

1"""Serply search engine implementation for Local Deep Research. 

2 

3Serply (https://serply.io) is a hosted Google SERP API. Its ``/v1/search``, 

4``/v1/news`` and ``/v1/scholar`` endpoints return Google web results, Google 

5News entries and Google Scholar articles, so one engine covers general web 

6search plus the two verticals. API reference: https://serply.io/docs. 

7""" 

8 

9import html 

10import re 

11from typing import Any, Dict, List, Literal, Optional 

12 

13import requests 

14from langchain_core.language_models import BaseLLM 

15 

16from ...constants import USER_AGENT 

17from ...security.safe_requests import safe_get 

18from ...security.secure_logging import logger 

19from ..rate_limiting import RateLimitError 

20from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity 

21 

22 

23class SerplySearchEngine(BaseSearchEngine): 

24 """Google search engine implementation using the Serply API.""" 

25 

26 # Mark as a public, generic (general web) search engine. 

27 is_public = True 

28 egress_sensitivity = Sensitivity.NON_SENSITIVE 

29 egress_exposure = Exposure.EXPOSING 

30 is_generic = True 

31 

32 #: Base URL for the Serply REST API. 

33 BASE_URL = "https://api.serply.io/v1" 

34 #: Endpoint path and result list key for each supported search type. 

35 SEARCH_TYPES = { 

36 "web": ("search", "results"), 

37 "news": ("news", "entries"), 

38 "scholar": ("scholar", "articles"), 

39 } 

40 #: Serply returns at most 100 results per request. 

41 MAX_RESULTS_CAP = 100 

42 #: HTTP request timeout in seconds. 

43 REQUEST_TIMEOUT = 30 

44 

45 def __init__( 

46 self, 

47 max_results: int = 10, 

48 search_type: Literal["web", "news", "scholar"] = "web", 

49 api_key: Optional[str] = None, 

50 llm: Optional[BaseLLM] = None, 

51 max_filtered_results: Optional[int] = None, 

52 settings_snapshot: Optional[Dict[str, Any]] = None, 

53 **kwargs, 

54 ): 

55 """Initialize the Serply search engine. 

56 

57 Args: 

58 max_results: Maximum number of search results to request. 

59 search_type: ``"web"`` (Google web search), ``"news"`` (Google 

60 News) or ``"scholar"`` (Google Scholar). 

61 api_key: Serply API key. May also be supplied via the 

62 ``search.engine.web.serply.api_key`` setting or the 

63 ``LDR_SEARCH_ENGINE_WEB_SERPLY_API_KEY`` environment variable. 

64 llm: Language model for relevance filtering. 

65 max_filtered_results: Maximum number of results to keep after 

66 relevance filtering. 

67 settings_snapshot: Settings snapshot for thread context. 

68 **kwargs: Remaining ``BaseSearchEngine`` parameters 

69 (``programmatic_mode``, ``include_full_content``, filters), 

70 forwarded to ``super().__init__``. 

71 """ 

72 super().__init__( 

73 llm=llm, 

74 max_filtered_results=max_filtered_results, 

75 max_results=max_results, 

76 settings_snapshot=settings_snapshot, 

77 **kwargs, 

78 ) 

79 if search_type not in self.SEARCH_TYPES: 

80 logger.warning( 

81 f"Unknown Serply search_type '{search_type}'; falling back " 

82 f"to 'web' (valid: {', '.join(self.SEARCH_TYPES)})" 

83 ) 

84 search_type = "web" 

85 self.search_type = search_type 

86 # Scholar results are academic papers, so route them through the 

87 # scientific pipeline (DOI enrichment, journal reputation filter) 

88 # instead of treating them as generic web pages. Instance-level so 

89 # one class can serve all three verticals. 

90 self.is_scientific = search_type == "scholar" 

91 

92 # Resolve API key from params, settings snapshot, or env var. 

93 self.api_key = self._resolve_api_key( 

94 api_key, 

95 "search.engine.web.serply.api_key", 

96 engine_name="Serply", 

97 settings_snapshot=settings_snapshot, 

98 ) 

99 

100 def _snippet(self, result: Dict[str, Any]) -> str: 

101 """Return the preview snippet for one Serply result.""" 

102 if self.search_type == "news": 

103 # Google News entries carry the summary as an HTML fragment. 

104 # Unescape entities first so encoded markup (``<b>``) 

105 # is also removed by the tag strip instead of surviving it. 

106 text = html.unescape(result.get("summary") or "") 

107 text = re.sub(r"<[^>]+>", "", text) 

108 return " ".join(text.split()) 

109 # Web results describe the page; Scholar results describe the 

110 # authors, venue and year. 

111 return result.get("description") or "" 

112 

113 def _get_previews(self, query: str) -> List[Dict[str, Any]]: 

114 """Get preview information from the Serply API. 

115 

116 Args: 

117 query: The search query. 

118 

119 Returns: 

120 List of preview dictionaries. 

121 """ 

122 logger.info("Getting search results from Serply API") 

123 

124 path, results_key = self.SEARCH_TYPES[self.search_type] 

125 

126 try: 

127 params = { 

128 "q": query, 

129 "num": min(self.max_results, self.MAX_RESULTS_CAP), 

130 } 

131 

132 # Apply rate limiting before the request. 

133 self._last_wait_time = self.rate_tracker.apply_rate_limit( 

134 self.engine_type 

135 ) 

136 

137 response = safe_get( 

138 f"{self.BASE_URL}/{path}/", 

139 params=params, 

140 headers={ 

141 "X-Api-Key": self.api_key, 

142 "Accept": "application/json", 

143 # Serply is fronted by Cloudflare, which can reject the 

144 # default python-requests User-Agent. 

145 "User-Agent": USER_AGENT, 

146 }, 

147 timeout=self.REQUEST_TIMEOUT, 

148 ) 

149 

150 # Detect rate limiting from the status code. 

151 self._raise_if_rate_limit(response.status_code) 

152 response.raise_for_status() 

153 

154 data = response.json() 

155 # The news endpoint ignores ``num``, so cap client-side too. 

156 items = (data.get(results_key) or [])[: self.max_results] 

157 

158 previews = [] 

159 for idx, result in enumerate(items): 

160 link = result.get("link") or "" 

161 preview = { 

162 "id": idx, 

163 "title": result.get("title", ""), 

164 "link": link, 

165 "snippet": self._snippet(result), 

166 "displayed_link": self._extract_display_link(link), 

167 "position": idx + 1, 

168 } 

169 if result.get("published"): 

170 preview["date"] = result["published"] 

171 previews.append(preview) 

172 

173 self._search_results = previews 

174 return previews 

175 

176 except RateLimitError: 

177 raise # Re-raise rate limit errors 

178 except requests.exceptions.RequestException as e: 

179 safe_msg = self._scrub_error(e) 

180 logger.warning(f"Error getting Serply API results: {safe_msg}") 

181 status_code = getattr( 

182 getattr(e, "response", None), "status_code", None 

183 ) 

184 if status_code is not None: 

185 self._raise_if_rate_limit(status_code) 

186 return [] 

187 except Exception as e: 

188 safe_msg = self._scrub_error(e) 

189 logger.warning( 

190 f"Unexpected error getting Serply API results: {safe_msg}" 

191 ) 

192 return []