Coverage for src/local_deep_research/research_library/downloaders/extraction/trafilatura_extractor.py: 94%

1""" 

2Trafilatura-based content extractor. 

3 

4Uses statistical + rule-based heuristics for boilerplate removal with 

5built-in language detection and optional markdown output. Trafilatura 

6internally falls back to readability and justext when its primary 

7heuristic fails, making it a strong standalone or first-pass extractor. 

8""" 

9 

10from typing import Optional 

11 

12from loguru import logger 

13 

14from ....utilities.lxml_thread_safety import parse_for_trafilatura 

15from .base import BaseExtractor 

16 

17 

18class TrafilaturaExtractor(BaseExtractor): 

19 """Extract content using trafilatura.""" 

20 

21 def __init__( 

22 self, 

23 output_format: str = "markdown", 

24 include_tables: bool = True, 

25 include_links: bool = False, 

26 include_comments: bool = False, 

27 include_formatting: bool = True, 

28 ): 

29 self.output_format = output_format 

30 self.include_tables = include_tables 

31 self.include_links = include_links 

32 self.include_comments = include_comments 

33 self.include_formatting = include_formatting 

34 

35 def extract(self, html: str) -> Optional[str]: 

36 if not html or not html.strip(): 

37 return None 

38 

39 try: 

40 import trafilatura 

41 except ImportError: 

42 logger.warning("trafilatura not installed — skipping extraction") 

43 return None 

44 

45 # Hand trafilatura a tree parsed with THIS thread's parser rather than 

46 # a raw string. Passing a string makes it parse through its own 

47 # module-level HTML_PARSER, which is how worker threads come to share 

48 # one libxml2 xmlDict and corrupt it (see utilities.lxml_thread_safety). 

49 # trafilatura.load_html() accepts an HtmlElement directly, and output 

50 # is byte-identical either way. Falls back to the string if our parse 

51 # fails, so a parse quirk degrades to the old behaviour, not to nothing. 

52 source = parse_for_trafilatura(html) 

53 if source is None: 53 ↛ 54line 53 didn't jump to line 54 because the condition on line 53 was never true

54 source = html 

55 

56 try: 

57 result = trafilatura.extract( 

58 source, 

59 output_format=self.output_format, 

60 include_tables=self.include_tables, 

61 include_links=self.include_links, 

62 include_comments=self.include_comments, 

63 include_formatting=self.include_formatting, 

64 ) 

65 return result if result and result.strip() else None 

66 except Exception: 

67 logger.exception("trafilatura extraction failed") 

68 return None