Coverage for src/local_deep_research/research_library/downloaders/biorxiv.py: 98%

1""" 

2bioRxiv/medRxiv PDF and Text Downloader 

3""" 

4 

5import re 

6from typing import Dict, Optional 

7from urllib.parse import urlparse 

8 

9import requests 

10from loguru import logger 

11 

12from ...security.ssrf_validator import redact_url_for_log 

13from ...security.egress.policy import PolicyDeniedError 

14from .base import BaseDownloader, ContentType, DownloadResult 

15 

16 

17class BioRxivDownloader(BaseDownloader): 

18 """Downloader for bioRxiv and medRxiv preprints.""" 

19 

20 def can_handle(self, url: str) -> bool: 

21 """Check if URL is from bioRxiv or medRxiv.""" 

22 try: 

23 hostname = urlparse(url).hostname 

24 if not hostname: 

25 return False 

26 return ( 

27 hostname == "biorxiv.org" 

28 or hostname.endswith(".biorxiv.org") 

29 or hostname == "medrxiv.org" 

30 or hostname.endswith(".medrxiv.org") 

31 ) 

32 except Exception: 

33 return False 

34 

35 def download( 

36 self, url: str, content_type: ContentType = ContentType.PDF 

37 ) -> Optional[bytes]: 

38 """Download content from bioRxiv/medRxiv.""" 

39 if content_type == ContentType.TEXT: 

40 return self._download_text(url) 

41 return self._download_pdf(url) 

42 

43 def download_with_result( 

44 self, url: str, content_type: ContentType = ContentType.PDF 

45 ) -> DownloadResult: 

46 """Download content and return detailed result with skip reason.""" 

47 if content_type == ContentType.TEXT: 

48 # Try to get text from page 

49 text = self._fetch_abstract_from_page(url) 

50 if text: 

51 return DownloadResult( 

52 content=text.encode("utf-8"), 

53 is_success=True, 

54 content_is_complete=False, 

55 ) 

56 

57 # Fallback to PDF extraction 

58 pdf_content = self._download_pdf(url) 

59 if pdf_content: 

60 extracted_text = self.extract_text_from_pdf(pdf_content) 

61 if extracted_text: 

62 return DownloadResult( 

63 content=extracted_text.encode("utf-8"), is_success=True 

64 ) 

65 

66 return DownloadResult( 

67 skip_reason="Could not extract text from bioRxiv/medRxiv article" 

68 ) 

69 # Try to download PDF 

70 pdf_url = self._convert_to_pdf_url(url) 

71 if not pdf_url: 

72 return DownloadResult( 

73 skip_reason="Invalid bioRxiv/medRxiv URL format" 

74 ) 

75 

76 logger.info( 

77 f"Downloading bioRxiv/medRxiv PDF from {redact_url_for_log(pdf_url)}" 

78 ) 

79 pdf_content = super()._download_pdf(pdf_url) 

80 

81 if pdf_content: 

82 return DownloadResult(content=pdf_content, is_success=True) 

83 # Check if it's a server issue or article doesn't exist 

84 try: 

85 response = self.session.head(url, timeout=5) 

86 if response.status_code == 404: 

87 return DownloadResult( 

88 skip_reason="Article not found on bioRxiv/medRxiv", 

89 status_code=404, 

90 ) 

91 if response.status_code >= 500: 

92 return DownloadResult( 

93 skip_reason="bioRxiv/medRxiv server temporarily unavailable", 

94 status_code=response.status_code, 

95 ) 

96 except requests.RequestException: 

97 logger.opt(exception=False).debug( 

98 "Failed to check bioRxiv/medRxiv URL: {}", 

99 redact_url_for_log(url), 

100 ) 

101 return DownloadResult( 

102 skip_reason="Failed to download PDF from bioRxiv/medRxiv" 

103 ) 

104 

105 def _download_pdf( 

106 self, url: str, headers: Optional[Dict[str, str]] = None 

107 ) -> Optional[bytes]: 

108 """Download PDF from bioRxiv/medRxiv.""" 

109 # Convert URL to PDF format 

110 pdf_url = self._convert_to_pdf_url(url) 

111 

112 if not pdf_url: 

113 logger.error( 

114 f"Could not convert to PDF URL: {redact_url_for_log(url)}" 

115 ) 

116 return None 

117 

118 logger.info( 

119 f"Downloading bioRxiv/medRxiv PDF from {redact_url_for_log(pdf_url)}" 

120 ) 

121 return super()._download_pdf(pdf_url) 

122 

123 def _download_text(self, url: str) -> Optional[bytes]: 

124 """Get text content from bioRxiv/medRxiv.""" 

125 # Try to get abstract and metadata from the HTML page 

126 text = self._fetch_abstract_from_page(url) 

127 if text: 

128 return text.encode("utf-8") 

129 

130 # Fallback: Download PDF and extract text 

131 pdf_content = self._download_pdf(url) 

132 if pdf_content: 

133 extracted_text = self.extract_text_from_pdf(pdf_content) 

134 if extracted_text: 

135 return extracted_text.encode("utf-8") 

136 

137 return None 

138 

139 def _convert_to_pdf_url(self, url: str) -> Optional[str]: 

140 """Convert bioRxiv/medRxiv URL to PDF URL.""" 

141 # Handle different URL patterns 

142 # Example: https://www.biorxiv.org/content/10.1101/2024.01.01.123456v1 

143 # Becomes: https://www.biorxiv.org/content/10.1101/2024.01.01.123456v1.full.pdf 

144 

145 # Remove any existing .full or .full.pdf 

146 base_url = re.sub(r"\.full(\.pdf)?$", "", url) 

147 

148 # Check if it's already a PDF URL 

149 if base_url.endswith(".pdf"): 

150 return base_url 

151 

152 # Add .full.pdf 

153 pdf_url = base_url.rstrip("/") + ".full.pdf" 

154 

155 # Handle content vs. content/early URLs 

156 return pdf_url.replace("/content/early/", "/content/") 

157 

158 def _fetch_abstract_from_page(self, url: str) -> Optional[str]: 

159 """Fetch abstract and metadata from bioRxiv/medRxiv page.""" 

160 try: 

161 # Request the HTML page 

162 response = self.session.get(url, timeout=10) 

163 

164 if response.status_code == 200: 

165 # Simple extraction using regex (avoiding BeautifulSoup dependency) 

166 html = response.text 

167 text_parts = [] 

168 

169 # Extract title 

170 title_match = re.search( 

171 r'<meta\s+name="DC\.Title"\s+content="([^"]+)"', html 

172 ) 

173 if title_match: 

174 text_parts.append(f"Title: {title_match.group(1)}") 

175 

176 # Extract authors 

177 author_match = re.search( 

178 r'<meta\s+name="DC\.Creator"\s+content="([^"]+)"', html 

179 ) 

180 if author_match: 

181 text_parts.append(f"Authors: {author_match.group(1)}") 

182 

183 # Extract abstract 

184 abstract_match = re.search( 

185 r'<meta\s+name="DC\.Description"\s+content="([^"]+)"', 

186 html, 

187 re.DOTALL, 

188 ) 

189 if abstract_match: 

190 abstract = abstract_match.group(1) 

191 # Clean up HTML entities 

192 abstract = abstract.replace("&lt;", "<").replace( 

193 "&gt;", ">" 

194 ) 

195 abstract = abstract.replace("&quot;", '"').replace( 

196 "&#39;", "'" 

197 ) 

198 abstract = abstract.replace("&amp;", "&") 

199 text_parts.append(f"\nAbstract:\n{abstract}") 

200 

201 if text_parts: 

202 logger.info( 

203 "Retrieved text content from bioRxiv/medRxiv page" 

204 ) 

205 return "\n".join(text_parts) 

206 

207 except PolicyDeniedError: 

208 raise 

209 except Exception as e: 

210 logger.opt(exception=False).debug( 

211 "Failed to fetch abstract from bioRxiv/medRxiv: {} ({})", 

212 redact_url_for_log(url), 

213 type(e).__name__, 

214 ) 

215 

216 return None