Coverage for src/local_deep_research/utilities/host_match.py: 100%

1"""Host membership shared by URL routing and source labels. 

2 

3Kept free of project imports so any module can use it: ``utilities/arxiv.py`` 

4and the content fetcher's ``URLClassifier`` both do, and a helper in 

5``url_utils.py`` would have pulled the security package into their import graph. 

6""" 

7 

8from urllib.parse import ParseResult 

9 

10 

11def http_authority_host(parsed: ParseResult) -> str | None: 

12 """Return the host an HTTP client would connect to, or ``None``. 

13 

14 The host is lower-cased and loses the root dot of a fully qualified name. 

15 ``None`` means urllib and the HTTP stack disagree about the authority, so 

16 no host can be trusted. That happens in two cases. urllib keeps a 

17 backslash inside the authority, while browsers and requests end the 

18 authority there, so ``https://evil.example\\@doi.org/`` has the host 

19 ``doi.org`` for urllib and connects to ``evil.example``. And a port that 

20 is not a number (``host:443.evil.example``) hides the rest of the 

21 authority from ``hostname``, while requests refuses the URL. 

22 """ 

23 if "\\" in parsed.netloc: 

24 return None 

25 try: 

26 _ = parsed.port 

27 except ValueError: 

28 return None 

29 host = parsed.hostname 

30 return host.rstrip(".") if host else None 

31 

32 

33def is_host_or_subdomain(hostname: str, domain: str) -> bool: 

34 """Return whether ``hostname`` is ``domain`` itself or a subdomain of it. 

35 

36 Both arguments are expected lower-cased, without port, userinfo or a 

37 trailing root dot. The dot boundary is the point of the check: a 

38 substring test also accepts ``doi.org.evil.example`` as ``doi.org``, and 

39 a suffix test without the dot accepts ``notdoi.org``. 

40 """ 

41 return hostname == domain or hostname.endswith(f".{domain}")