Coverage for src/local_deep_research/research_library/downloaders/semantic_scholar.py: 97%

1""" 

2Semantic Scholar PDF Downloader 

3 

4Downloads PDFs from Semantic Scholar using their API to find open access PDFs. 

5""" 

6 

7import re 

8from typing import Optional 

9from urllib.parse import urlparse 

10 

11import requests 

12from loguru import logger 

13 

14from ...security.log_sanitizer import scrub_error 

15from ...security.ssrf_validator import redact_url_for_log 

16from .base import BaseDownloader, ContentType, DownloadResult 

17 

18 

19class SemanticScholarDownloader(BaseDownloader): 

20 """Downloader for Semantic Scholar papers with open access PDF support.""" 

21 

22 def __init__(self, timeout: int = 30, api_key: Optional[str] = None): 

23 """ 

24 Initialize Semantic Scholar downloader. 

25 

26 Args: 

27 timeout: Request timeout in seconds 

28 api_key: Optional Semantic Scholar API key for higher rate limits 

29 """ 

30 super().__init__(timeout) 

31 self.api_key = api_key 

32 self.base_api_url = "https://api.semanticscholar.org/graph/v1" 

33 

34 def can_handle(self, url: str) -> bool: 

35 """Check if URL is from Semantic Scholar.""" 

36 try: 

37 hostname = urlparse(url).hostname 

38 return bool( 

39 hostname 

40 and ( 

41 hostname == "semanticscholar.org" 

42 or hostname.endswith(".semanticscholar.org") 

43 ) 

44 ) 

45 except (ValueError, AttributeError, TypeError): 

46 return False 

47 

48 def download( 

49 self, url: str, content_type: ContentType = ContentType.PDF 

50 ) -> Optional[bytes]: 

51 """Download content from Semantic Scholar.""" 

52 result = self.download_with_result(url, content_type) 

53 return result.content if result.is_success else None 

54 

55 def download_with_result( 

56 self, url: str, content_type: ContentType = ContentType.PDF 

57 ) -> DownloadResult: 

58 """Download PDF and return detailed result with skip reason.""" 

59 # Only support PDF downloads for now 

60 if content_type != ContentType.PDF: 

61 return DownloadResult( 

62 skip_reason="Text extraction not yet supported for Semantic Scholar" 

63 ) 

64 

65 # Extract paper ID from URL 

66 paper_id = self._extract_paper_id(url) 

67 if not paper_id: 

68 return DownloadResult( 

69 skip_reason="Invalid Semantic Scholar URL - could not extract paper ID" 

70 ) 

71 

72 logger.info(f"Looking up Semantic Scholar paper: {paper_id}") 

73 

74 # Get paper details from API to find PDF URL 

75 pdf_url = self._get_pdf_url(paper_id) 

76 

77 if not pdf_url: 77 ↛ 78line 77 didn't jump to line 78 because the condition on line 77 was never true

78 return DownloadResult( 

79 skip_reason="Not open access - subscription required" 

80 ) 

81 

82 # Download the PDF from the open access URL 

83 logger.info( 

84 f"Downloading open access PDF from: {redact_url_for_log(pdf_url)}" 

85 ) 

86 pdf_content = super()._download_pdf(pdf_url) 

87 

88 if pdf_content: 

89 return DownloadResult(content=pdf_content, is_success=True) 

90 return DownloadResult( 

91 skip_reason="Open access PDF URL found but download failed" 

92 ) 

93 

94 def _extract_paper_id(self, url: str) -> Optional[str]: 

95 """ 

96 Extract Semantic Scholar paper ID from URL. 

97 

98 Handles formats like: 

99 - https://www.semanticscholar.org/paper/abc123... 

100 - https://www.semanticscholar.org/paper/Title-Here/abc123... 

101 

102 Returns: 

103 Paper ID (hash) or None if not found 

104 """ 

105 # Use urlparse for more robust URL handling (handles query strings, fragments) 

106 parsed = urlparse(url) 

107 if not parsed.netloc or "semanticscholar.org" not in parsed.netloc: 

108 return None 

109 

110 # Extract paper ID from path (40 character hex hash) 

111 # Handles /paper/{hash} or /paper/{title}/{hash} 

112 path = parsed.path 

113 match = re.search(r"/paper/(?:[^/]+/)?([a-f0-9]{40})", path) 

114 return match.group(1) if match else None 

115 

116 def _get_pdf_url(self, paper_id: str) -> Optional[str]: 

117 """ 

118 Get open access PDF URL from Semantic Scholar API. 

119 

120 Args: 

121 paper_id: Semantic Scholar paper ID (hash) 

122 

123 Returns: 

124 PDF URL if available, None otherwise 

125 """ 

126 try: 

127 # Construct API request 

128 api_url = f"{self.base_api_url}/paper/{paper_id}" 

129 params = {"fields": "openAccessPdf"} 

130 

131 # Add API key header if available 

132 headers = {} 

133 if self.api_key: 

134 headers["x-api-key"] = self.api_key 

135 

136 # Make API request 

137 response = self.session.get( 

138 api_url, params=params, headers=headers, timeout=self.timeout 

139 ) 

140 

141 if response.status_code == 200: 

142 data = response.json() 

143 

144 # Extract PDF URL from openAccessPdf field 

145 open_access_pdf = data.get("openAccessPdf") 

146 if open_access_pdf and isinstance(open_access_pdf, dict): 

147 pdf_url = open_access_pdf.get("url") 

148 if pdf_url: 148 ↛ 155line 148 didn't jump to line 155 because the condition on line 148 was always true

149 logger.info( 

150 f"Found open access PDF for paper {paper_id}: " 

151 f"{redact_url_for_log(str(pdf_url))}" 

152 ) 

153 return str(pdf_url) 

154 

155 logger.info( 

156 f"No open access PDF available for paper {paper_id}" 

157 ) 

158 return None 

159 

160 if response.status_code == 404: 

161 logger.warning( 

162 f"Paper not found in Semantic Scholar: {paper_id}" 

163 ) 

164 return None 

165 logger.warning( 

166 f"Semantic Scholar API error: {response.status_code}" 

167 ) 

168 return None 

169 

170 # Both handlers keep the scrubbed message rather than dropping the 

171 # API error -- the ``safe_msg`` shape the sibling ``openalex`` 

172 # downloader uses for the same handler pair. No traceback: these 

173 # frames hold the Semantic Scholar API key (``self.api_key`` and the 

174 # ``headers`` dict built from it), which loguru's ``diagnose`` 

175 # renders for every frame-local, so ``scrub_error`` is passed that 

176 # key as a known literal on top of its shape-based scrubbing. 

177 # 

178 # WARNING rather than the ERROR main's ``logger.exception`` logged 

179 # at: open-access PDF discovery is optional -- the caller falls back 

180 # to the landing page -- so a network blip or a malformed response 

181 # here is routine, not an outage. 

182 except requests.exceptions.RequestException as exc: 

183 safe_msg = scrub_error(exc, self.api_key) 

184 logger.opt(exception=False).warning( 

185 "Failed to query Semantic Scholar API for paper {}: {}", 

186 paper_id, 

187 safe_msg, 

188 ) 

189 return None 

190 except ValueError as exc: 

191 # JSON decode errors are expected runtime errors 

192 safe_msg = scrub_error(exc, self.api_key) 

193 logger.opt(exception=False).warning( 

194 "Failed to parse Semantic Scholar API response for paper " 

195 "{}: {}", 

196 paper_id, 

197 safe_msg, 

198 ) 

199 return None 

200 # Note: KeyError and TypeError are not caught - they indicate programming 

201 # bugs that should propagate for debugging