Coverage for src/local_deep_research/utilities/download_url.py: 91%

1"""URL identity shared by download writers, readers and research deletion. 

2 

3No database, service or web imports are needed to load these helpers. 

4""" 

5 

6import hashlib 

7import re 

8 

9from loguru import logger 

10 

11 

12def normalize_url_for_hash(url: str) -> str: 

13 """Canonical normalization for URL-identity hashing (issue #6758 B2). 

14 

15 Composes the shared :func:`canonical_url_key` (strips tracking query 

16 params, fragments, default ports, and userinfo; lowercases scheme and 

17 host) with the historical scheme/``www``/trailing-slash stripping, 

18 query-param sorting, and full lowercasing, so ``http`` vs ``https``, 

19 ``www`` variants, tracking-param decor, and param order all hash alike. 

20 

21 Lowercasing happens BEFORE query-param sorting: ``sort(raw-case).lower() 

22 != sort(lower(raw))`` whenever mixed-case query keys change relative 

23 order (e.g. ``?q=x&Lang=fr&UserId=1``), and the tracker writer 

24 re-normalizes an already-lowercased string — sorting after lowering on 

25 one path but before lowering on the other yields different orders and 

26 different hashes (issue #6760 follow-up). Sorting the lowered form is 

27 deterministic on both paths and keeps the function idempotent. 

28 

29 The ``canonical_url_key`` import is lazy: this module is imported at 

30 load time across both the database and research-library stacks. Defer 

31 loading the utilities/security stack until normalization is needed. 

32 """ 

33 from .url_utils import canonical_url_key 

34 

35 try: 

36 normalized = canonical_url_key(url) 

37 except Exception: 

38 logger.warning("canonical_url_key failed, hashing raw URL") 

39 normalized = url if isinstance(url, str) else "" 

40 normalized = re.sub(r"^https?://", "", normalized or "") 

41 normalized = re.sub(r"^www\.", "", normalized) 

42 normalized = normalized.rstrip("/") 

43 # Lowercase first so the sort below is on the lowered form (see 

44 # docstring): otherwise mixed-case query keys sort differently before 

45 # vs after lowering and a double-normalized writer diverges from a 

46 # single-normalized reader. 

47 normalized = normalized.lower() 

48 if "?" in normalized: 

49 base, query = normalized.split("?", 1) 

50 params = sorted(query.split("&")) 

51 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: URL query-param sorting only; this module builds no SQL 

52 normalized = f"{base}?{'&'.join(params)}" 

53 return normalized 

54 

55 

56def get_url_hash(url: str) -> str: 

57 """ 

58 Generate a SHA256 hash of a URL. 

59 

60 Args: 

61 url: The URL to hash 

62 

63 Returns: 

64 The SHA256 hash of the canonical normalized URL (issue #6758 B2) 

65 """ 

66 return hashlib.sha256(normalize_url_for_hash(url).encode()).hexdigest() 

67 

68 

69def _legacy_normalize_download_service(url: str) -> str: 

70 """Pre-B2 (#6758) DownloadService._normalize_url spelling. 

71 

72 Stripped scheme/``www``/trailing-slash, sorted raw ``&`` query pairs, 

73 lowercased — but did NOT strip tracking params, fragments, default 

74 ports, or userinfo. Rows written before B2 still carry this hash. 

75 """ 

76 if not isinstance(url, str) or not url: 76 ↛ 77line 76 didn't jump to line 77 because the condition on line 76 was never true

77 return "" 

78 normalized = re.sub(r"^https?://", "", url) 

79 normalized = re.sub(r"^www\.", "", normalized) 

80 normalized = normalized.rstrip("/") 

81 if "?" in normalized: 

82 base, query = normalized.split("?", 1) 

83 params = sorted(query.split("&")) 

84 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: URL query-param sorting only; this module builds no SQL 

85 normalized = f"{base}?{'&'.join(params)}" 

86 return normalized.lower() 

87 

88 

89def _legacy_normalize_library_service(url: str) -> str: 

90 """Pre-B2 (#6758) LibraryService._get_url_hash spelling. 

91 

92 Same stripping as the download-service variant but WITHOUT query-param 

93 sorting. A separate candidate is required for multi-param URLs whose 

94 order differed. 

95 """ 

96 if not isinstance(url, str) or not url: 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true

97 return "" 

98 normalized = re.sub(r"^https?://", "", url) 

99 normalized = re.sub(r"^www\.", "", normalized) 

100 normalized = normalized.rstrip("/") 

101 return normalized.lower() 

102 

103 

104def get_legacy_url_hashes(url: str) -> list[str]: 

105 """Return pre-B2 (#6758) url_hash candidates for ``url``. 

106 

107 Three legacy spellings existed in production: 

108 - raw ``sha256(url.lower())`` (cascade_helper looked up trackers with 

109 a bare ``get_url_hash`` that performed no stripping); 

110 - download-service normalized (scheme/``www``/trailing-slash strip + 

111 query sort); 

112 - library-service normalized (same strip, no sort). 

113 

114 Used only as a read-compat fallback — new rows are always written with 

115 the canonical :func:`get_url_hash`. Callers must try the canonical hash 

116 first and fall back to these, never write them. 

117 """ 

118 if not isinstance(url, str) or not url: 118 ↛ 119line 118 didn't jump to line 119 because the condition on line 118 was never true

119 return [] 

120 candidates: list[str] = [] 

121 raw_hash = hashlib.sha256(url.lower().encode()).hexdigest() 

122 candidates.append(raw_hash) 

123 dl_hash = hashlib.sha256( 

124 _legacy_normalize_download_service(url).encode() 

125 ).hexdigest() 

126 if dl_hash not in candidates: 126 ↛ 128line 126 didn't jump to line 128 because the condition on line 126 was always true

127 candidates.append(dl_hash) 

128 lib_hash = hashlib.sha256( 

129 _legacy_normalize_library_service(url).encode() 

130 ).hexdigest() 

131 if lib_hash not in candidates: 

132 candidates.append(lib_hash) 

133 return candidates 

134 

135 

136def get_url_hash_candidates(url: str) -> list[str]: 

137 """Return ``[canonical, *legacy]`` url_hash candidates for ``url``. 

138 

139 B2 (#6758) re-keys every URL that ``canonical_url_key`` mutates 

140 (tracking params, fragments, default ports, userinfo). Pre-upgrade 

141 ``download_tracker`` rows still carry a legacy hash, so a lookup by 

142 the canonical hash alone misses them — which previously drove 

143 duplicate tracker rows and, worse, ``sync_library_with_filesystem``'s 

144 "No tracker or path" branch deleting the document. Query 

145 ``DownloadTracker.url_hash.in_(candidates)`` (canonical first) to stay 

146 compatible with mixed-scheme tables without a data migration. 

147 """ 

148 canonical = get_url_hash(url) 

149 out = [canonical] 

150 for legacy in get_legacy_url_hashes(url): 

151 if legacy != canonical and legacy not in out: 

152 out.append(legacy) 

153 return out