Coverage for src/local_deep_research/utilities/lxml_thread_safety.py: 94%

1"""Per-thread lxml parsers, so concurrent HTML extraction cannot corrupt the heap. 

2 

3Why this exists 

4--------------- 

5A research run reads several pages at once — one worker thread per page — and 

6each thread runs the extraction pipeline, which parses HTML through 

7trafilatura, newspaper4k, readabilipy and justext. All four reach lxml, and 

8therefore libxml2. 

9 

10lxml keeps a *per-thread* string dictionary (libxml2's ``xmlDict``). On entry 

11to a parse it rebinds the parser context's dict to the calling thread's dict 

12(``_ParserDictionaryContext.initThreadDictRef``, ``lxml/parser.pxi``), and a 

13thread that has no dict yet **adopts whatever dict the parser is currently 

14carrying**. Because trafilatura parses through a module-level parser 

15(``trafilatura.utils.HTML_PARSER``) and newspaper4k/readabilipy parse through 

16lxml's own module-level ``lxml.html.html_parser``, worker threads end up 

17adopting *each other's* dict and then interning strings into one ``xmlDict`` 

18concurrently. ``xmlDict`` is not thread-safe, so its hash table gets corrupted 

19and the interpreter dies with SIGABRT — ``double free or corruption (out)`` 

20raised from inside ``htmlParseDocument``, taking every user's session and all 

21in-flight research down with it. 

22 

23lxml's own per-parser lock does not prevent this: the racing parses run on 

24*different* parser objects that merely share a dict, so the lock is never 

25contended by them. 

26 

27What this does 

28-------------- 

29lxml's FAQ prescribes "create a parser for each thread yourself". Each library 

30needs a different route to get there: 

31 

32* ``lxml.html.fromstring`` is wrapped so that a call passing no explicit 

33 ``parser=`` receives *this thread's* parser rather than the process-global 

34 one. newspaper4k (``newspaper/parsers.py``) and readabilipy 

35 (``readabilipy/extractors/extract_element.py``) both call it as a module 

36 attribute, so the wrapper reaches them. Anything else in the process that 

37 calls ``lxml.html.fromstring`` is covered for free. 

38* trafilatura cannot be reached that way — it ``from``-imports ``fromstring`` 

39 and always passes its own ``HTML_PARSER`` explicitly. Instead callers hand 

40 it an already-parsed tree via :func:`parse_for_trafilatura`; its 

41 ``load_html`` accepts an ``HtmlElement`` directly. 

42* justext already builds a fresh parser per call and needs nothing. 

43 

44Extraction output was verified byte-identical before and after this change on 

45the three real pages from the crash (3237 / 21578 / 26882 characters). 

46 

47Note the parsers below are built from ``lxml.html.HTMLParser``, never 

48``lxml.etree.HTMLParser``. The etree variant yields plain ``_Element`` objects, 

49which fail trafilatura's ``isinstance(..., HtmlElement)`` check in 

50``load_html`` — it then returns None and extraction silently yields empty text 

51rather than raising. 

52""" 

53 

54import threading 

55from typing import Any, Optional 

56 

57import lxml.html 

58from loguru import logger 

59 

60# One parser per thread, per configuration. Parsers are cheap; a worker thread 

61# builds its two parsers once and reuses them for its lifetime. 

62_local = threading.local() 

63 

64_install_lock = threading.Lock() 

65_installed = False 

66 

67 

68def get_thread_parser() -> lxml.html.HTMLParser: 

69 """This thread's general-purpose HTML parser. 

70 

71 Deliberately constructed with lxml's defaults so that replacing the 

72 process-global ``lxml.html.html_parser`` changes *which* parser object is 

73 used and nothing else about how the document is parsed. 

74 """ 

75 parser = getattr(_local, "html_parser", None) 

76 if parser is None: 

77 parser = lxml.html.HTMLParser() 

78 _local.html_parser = parser 

79 return parser 

80 

81 

82def get_thread_trafilatura_parser() -> lxml.html.HTMLParser: 

83 """This thread's parser configured exactly as trafilatura configures its own. 

84 

85 Mirrors ``trafilatura.utils.HTML_PARSER``. The options must match, or the 

86 tree we hand trafilatura would not be the tree it would have built for 

87 itself, and extraction output could drift. 

88 """ 

89 parser = getattr(_local, "trafilatura_parser", None) 

90 if parser is None: 

91 parser = lxml.html.HTMLParser( 

92 collect_ids=False, 

93 default_doctype=False, 

94 encoding="utf-8", 

95 remove_comments=True, 

96 remove_pis=True, 

97 ) 

98 _local.trafilatura_parser = parser 

99 return parser 

100 

101 

102def parse_for_trafilatura(html: str) -> Optional[Any]: 

103 """Parse ``html`` with this thread's parser, for handing to trafilatura. 

104 

105 Returns an ``HtmlElement``, or None if the document could not be parsed — 

106 in which case the caller should fall back to passing the raw string, since 

107 trafilatura's own error handling is more forgiving than ours needs to be. 

108 """ 

109 if not html or not html.strip(): 

110 return None 

111 try: 

112 return lxml.html.fromstring( 

113 html, parser=get_thread_trafilatura_parser() 

114 ) 

115 except Exception: 

116 logger.debug( 

117 "Per-thread parse failed; caller should fall back to the raw string" 

118 ) 

119 return None 

120 

121 

122def install_per_thread_html_parsers() -> bool: 

123 """Route parser-less ``lxml.html.fromstring`` calls onto per-thread parsers. 

124 

125 Idempotent and safe to call from any thread. Returns True if this call 

126 installed the wrapper, False if it was already in place. 

127 

128 Must run before worker threads start parsing — importing it at extraction 

129 module import time is early enough, since nothing parses before that. 

130 """ 

131 global _installed 

132 with _install_lock: 

133 if _installed: 

134 return False 

135 

136 original = lxml.html.fromstring 

137 

138 def fromstring(html, base_url=None, parser=None, **kwargs): 

139 if parser is None: 

140 parser = get_thread_parser() 

141 return original(html, base_url=base_url, parser=parser, **kwargs) 

142 

143 # Keep a handle on the original so the wrapper is inspectable and a 

144 # second install() can detect it rather than double-wrapping. 

145 fromstring.__wrapped__ = original 

146 fromstring.__doc__ = ( 

147 "lxml.html.fromstring, defaulting to a per-thread parser. " 

148 "See local_deep_research.utilities.lxml_thread_safety." 

149 ) 

150 

151 lxml.html.fromstring = fromstring 

152 _installed = True 

153 logger.debug( 

154 "Installed per-thread lxml.html parsers " 

155 "(guards against concurrent xmlDict corruption)" 

156 ) 

157 return True 

158 

159 

160def is_installed() -> bool: 

161 """Whether the per-thread parser wrapper is currently active.""" 

162 return getattr(lxml.html.fromstring, "__wrapped__", None) is not None