Coverage for src/local_deep_research/security/url_builder.py: 95%

1""" 

2URL building utilities for security and application use. 

3 

4Provides centralized URL construction logic that can be reused 

5throughout the application for consistent URL handling. 

6""" 

7 

8import re 

9from typing import Optional, Union 

10from urllib.parse import urlparse 

11from loguru import logger 

12 

13from .ssrf_validator import redact_url_for_log 

14 

15 

16class URLBuilderError(Exception): 

17 """Raised when URL construction fails.""" 

18 

19 pass 

20 

21 

22def normalize_bind_address(host: str) -> str: 

23 """ 

24 Convert bind addresses to URL-friendly hostnames. 

25 

26 Args: 

27 host: Host address from settings (may include bind addresses) 

28 

29 Returns: 

30 URL-friendly hostname 

31 """ 

32 # Convert bind-all addresses to localhost for URLs 

33 # Replace a bind-all address with localhost for a client URL. 

34 if host in ("0.0.0.0", "::"): # nosec B104 

35 return "localhost" 

36 return host 

37 

38 

39def build_base_url_from_settings( 

40 external_url: Optional[str] = None, 

41 host: Optional[str] = None, 

42 port: Optional[Union[str, int]] = None, 

43 fallback_base: str = "http://localhost:5000", 

44) -> str: 

45 """ 

46 Build a base URL from application settings with intelligent fallbacks. 

47 

48 This function handles the common pattern of building application URLs 

49 from various configuration sources with proper normalization. 

50 

51 Args: 

52 external_url: Pre-configured external URL (highest priority) 

53 host: Hostname/IP address (used if external_url not provided) 

54 port: Port number (used with host if external_url not provided) 

55 fallback_base: Final fallback URL if nothing else is available 

56 

57 Returns: 

58 Complete base URL (e.g., "https://myapp.com" or "http://localhost:5000") 

59 

60 Raises: 

61 URLBuilderError: If URL construction fails 

62 """ 

63 try: 

64 # Try external URL first (highest priority) 

65 if external_url and external_url.strip(): 

66 base_url = external_url.strip().rstrip("/") 

67 logger.debug( 

68 f"Using configured external URL: {redact_url_for_log(base_url)}" 

69 ) 

70 return base_url 

71 

72 # Try to construct from host and port 

73 if host and port: 

74 normalized_host = normalize_bind_address(host) 

75 

76 # Use HTTP for host/port combinations (typically internal server addresses) 

77 # For external URLs, users should configure external_url setting instead 

78 base_url = f"http://{normalized_host}:{int(port)}" # DevSkim: ignore DS137138 

79 logger.debug( 

80 f"Constructed URL from host/port: {redact_url_for_log(base_url)}" 

81 ) 

82 return base_url 

83 

84 # Final fallback 

85 base_url = fallback_base.rstrip("/") 

86 logger.debug(f"Using fallback URL: {redact_url_for_log(base_url)}") 

87 return base_url 

88 

89 except Exception as e: 

90 raise URLBuilderError(f"Failed to build base URL: {e}") 

91 

92 

93def build_full_url( 

94 base_url: str, 

95 path: str, 

96 validate: bool = True, 

97 allowed_schemes: Optional[list] = None, 

98) -> str: 

99 """ 

100 Build a complete URL from base URL and path. 

101 

102 Args: 

103 base_url: Base URL (e.g., "https://myapp.com") 

104 path: Path to append (e.g., "/research/123") 

105 validate: Whether to validate the resulting URL 

106 allowed_schemes: List of allowed URL schemes (default: ["http", "https"]) 

107 

108 Returns: 

109 Complete URL (e.g., "https://myapp.com/research/123") 

110 

111 Raises: 

112 URLBuilderError: If URL construction or validation fails 

113 """ 

114 try: 

115 # Ensure path starts with / 

116 if not path.startswith("/"): 

117 path = f"/{path}" 

118 

119 # Ensure base URL doesn't end with / 

120 base_url = base_url.rstrip("/") 

121 

122 # Construct full URL 

123 full_url = f"{base_url}{path}" 

124 

125 if validate: 

126 validate_constructed_url(full_url, allowed_schemes) 

127 

128 return full_url 

129 

130 except Exception as e: 

131 raise URLBuilderError(f"Failed to build full URL: {e}") 

132 

133 

134def validate_constructed_url( 

135 url: str, allowed_schemes: Optional[list] = None 

136) -> bool: 

137 """ 

138 Validate a constructed URL. 

139 

140 Args: 

141 url: URL to validate 

142 allowed_schemes: List of allowed schemes (default: ["http", "https"]) 

143 

144 Returns: 

145 True if valid 

146 

147 Raises: 

148 URLBuilderError: If URL is invalid 

149 """ 

150 if not url or not isinstance(url, str): 

151 raise URLBuilderError("URL must be a non-empty string") 

152 

153 try: 

154 parsed = urlparse(url) 

155 except Exception as e: 

156 raise URLBuilderError(f"Failed to parse URL: {e}") 

157 

158 # Check scheme 

159 if not parsed.scheme: 

160 raise URLBuilderError("URL must have a scheme") 

161 

162 if allowed_schemes and parsed.scheme not in allowed_schemes: 

163 raise URLBuilderError( 

164 f"URL scheme '{parsed.scheme}' not in allowed schemes: {allowed_schemes}" 

165 ) 

166 

167 # Check hostname 

168 if not parsed.netloc: 

169 raise URLBuilderError("URL must have a hostname") 

170 

171 return True 

172 

173 

174def mask_sensitive_url(url: str) -> str: 

175 """ 

176 Mask sensitive parts of a URL for secure logging. 

177 

178 This function masks passwords, webhook tokens, and other sensitive 

179 information in URLs to prevent accidental exposure in logs. 

180 

181 Args: 

182 url: URL to mask 

183 

184 Returns: 

185 URL with sensitive parts replaced with *** 

186 """ 

187 # A non-string never reaches the masking logic usefully: urlparse(None) 

188 # and urlparse([]) do NOT raise, they return empty byte-ish parts, so the 

189 # function used to emit nonsense like "b''://b''b''" -- unmasked and 

190 # meaningless. Non-strings also broke the except-branch, which does 

191 # url.split(':') and raises AttributeError from inside the very helper 

192 # that exists to make logging an unexpected value safe. There is no 

193 # scheme worth preserving here, so mask the whole thing. 

194 if not isinstance(url, str): 

195 return "***" 

196 

197 try: 

198 parsed = urlparse(url) 

199 

200 # Mask password if present 

201 if parsed.password: 

202 netloc = parsed.netloc.replace(parsed.password, "***") 

203 else: 

204 netloc = parsed.netloc 

205 

206 # Mask path tokens (common in webhooks) 

207 path = parsed.path 

208 if path: 

209 # Replace long alphanumeric tokens with *** 

210 path = re.sub( 

211 r"/[a-zA-Z0-9_-]{20,}", 

212 "/***", 

213 path, 

214 ) 

215 

216 # Reconstruct URL 

217 masked = f"{parsed.scheme}://{netloc}{path}" 

218 if parsed.query: 

219 masked += "?***" 

220 

221 return masked 

222 

223 except Exception: 

224 # If parsing fails, return a generic mask. ``url`` is a str by the 

225 # time it reaches here -- non-strings returned "***" above -- so the 

226 # split cannot itself raise. That matters: this handler exists to 

227 # make logging an unexpected value safe, so it must never be the 

228 # thing that fails. A value with no scheme masks entirely rather 

229 # than emitting a bare "://***". 

230 scheme = url.split(":", 1)[0] 

231 return f"{scheme}://***" if scheme else "***"