Coverage for src/local_deep_research/utilities/lxml_thread_safety.py: 94%

1"""Per-thread lxml parsers, so concurrent HTML extraction cannot corrupt the heap. 

2 

3Why this exists 

4--------------- 

5A research run reads several pages at once — one worker thread per page — and 

6each thread runs the extraction pipeline, which parses HTML through 

7trafilatura, newspaper4k, readabilipy and justext. All four reach lxml, and 

8therefore libxml2. 

9 

10lxml keeps a *per-thread* string dictionary (libxml2's ``xmlDict``). On entry 

11to a parse it rebinds the parser context's dict to the calling thread's dict 

12(``_ParserDictionaryContext.initThreadDictRef``, ``lxml/parser.pxi``), and a 

13thread that has no dict yet **adopts whatever dict the parser is currently 

14carrying**. Because trafilatura parses through a module-level parser 

15(``trafilatura.utils.HTML_PARSER``) and newspaper4k/readabilipy parse through 

16lxml's own module-level ``lxml.html.html_parser``, worker threads end up 

17adopting *each other's* dict and then interning strings into one ``xmlDict`` 

18concurrently. ``xmlDict`` is not thread-safe, so its hash table gets corrupted 

19and the interpreter dies with SIGABRT — ``double free or corruption (out)`` 

20raised from inside ``htmlParseDocument``, taking every user's session and all 

21in-flight research down with it. 

22 

23lxml's own per-parser lock does not prevent this: the racing parses run on 

24*different* parser objects that merely share a dict, so the lock is never 

25contended by them. 

26 

27What this does 

28-------------- 

29lxml's FAQ prescribes "create a parser for each thread yourself". Each library 

30needs a different route to get there: 

31 

32* ``lxml.html.fromstring`` is wrapped so that a call passing no explicit 

33 ``parser=`` receives *this thread's* parser rather than the process-global 

34 one. newspaper4k (``newspaper/parsers.py``) and readabilipy 

35 (``readabilipy/extractors/extract_element.py``) both call it as a module 

36 attribute, so the wrapper reaches them. Anything else in the process that 

37 calls ``lxml.html.fromstring`` is covered for free. 

38* trafilatura cannot be reached that way — it ``from``-imports ``fromstring`` 

39 and always passes its own ``HTML_PARSER`` explicitly. Instead callers hand 

40 it an already-parsed tree via :func:`parse_for_trafilatura`; its 

41 ``load_html`` accepts an ``HtmlElement`` directly. 

42* justext already builds a fresh parser per call and needs nothing. 

43 

44Extraction output was verified byte-identical before and after this change on 

45the three real pages from the crash (3237 / 21578 / 26882 characters). 

46 

47Note the parsers below are built from ``lxml.html.HTMLParser``, never 

48``lxml.etree.HTMLParser``. The etree variant yields plain ``_Element`` objects, 

49which fail trafilatura's ``isinstance(..., HtmlElement)`` check in 

50``load_html`` — it then returns None and extraction silently yields empty text 

51rather than raising. 

52""" 

53 

54import threading 

55from typing import Any, Optional 

56 

57import lxml.html 

58from loguru import logger 

59 

60# One parser per thread, per configuration. Parsers are cheap; a worker thread 

61# builds its two parsers once and reuses them for its lifetime. The 

62# short-lived per-call threads used to bound the Readability.js subprocess 

63# (see downloaders/extraction/readability_extractor) do not add to that: they 

64# run only readabilipy's ``use_readability=True`` branch, which shells out to 

65# node and never reaches ``lxml.html.fromstring`` — that call lives in the 

66# pure-Python branch, which runs on the caller's own thread. 

67_local = threading.local() 

68 

69_install_lock = threading.Lock() 

70_installed = False 

71 

72 

73def get_thread_parser() -> lxml.html.HTMLParser: 

74 """This thread's general-purpose HTML parser. 

75 

76 Deliberately constructed with lxml's defaults so that replacing the 

77 process-global ``lxml.html.html_parser`` changes *which* parser object is 

78 used and nothing else about how the document is parsed. 

79 """ 

80 parser = getattr(_local, "html_parser", None) 

81 if parser is None: 

82 parser = lxml.html.HTMLParser() 

83 _local.html_parser = parser 

84 return parser 

85 

86 

87def get_thread_trafilatura_parser() -> lxml.html.HTMLParser: 

88 """This thread's parser configured exactly as trafilatura configures its own. 

89 

90 Mirrors ``trafilatura.utils.HTML_PARSER``. The options must match, or the 

91 tree we hand trafilatura would not be the tree it would have built for 

92 itself, and extraction output could drift. 

93 """ 

94 parser = getattr(_local, "trafilatura_parser", None) 

95 if parser is None: 

96 parser = lxml.html.HTMLParser( 

97 collect_ids=False, 

98 default_doctype=False, 

99 encoding="utf-8", 

100 remove_comments=True, 

101 remove_pis=True, 

102 ) 

103 _local.trafilatura_parser = parser 

104 return parser 

105 

106 

107def parse_for_trafilatura(html: str) -> Optional[Any]: 

108 """Parse ``html`` with this thread's parser, for handing to trafilatura. 

109 

110 Returns an ``HtmlElement``, or None if the document could not be parsed — 

111 in which case the caller should fall back to passing the raw string, since 

112 trafilatura's own error handling is more forgiving than ours needs to be. 

113 """ 

114 if not html or not html.strip(): 

115 return None 

116 try: 

117 return lxml.html.fromstring( 

118 html, parser=get_thread_trafilatura_parser() 

119 ) 

120 except Exception: 

121 logger.debug( 

122 "Per-thread parse failed; caller should fall back to the raw string" 

123 ) 

124 return None 

125 

126 

127def install_per_thread_html_parsers() -> bool: 

128 """Route parser-less ``lxml.html.fromstring`` calls onto per-thread parsers. 

129 

130 Idempotent and safe to call from any thread. Returns True if this call 

131 installed the wrapper, False if it was already in place. 

132 

133 Must run before worker threads start parsing — importing it at extraction 

134 module import time is early enough, since nothing parses before that. 

135 """ 

136 global _installed 

137 with _install_lock: 

138 if _installed: 

139 return False 

140 

141 original = lxml.html.fromstring 

142 

143 def fromstring(html, base_url=None, parser=None, **kwargs): 

144 if parser is None: 

145 parser = get_thread_parser() 

146 return original(html, base_url=base_url, parser=parser, **kwargs) 

147 

148 # Keep a handle on the original so the wrapper is inspectable and a 

149 # second install() can detect it rather than double-wrapping. 

150 fromstring.__wrapped__ = original 

151 fromstring.__doc__ = ( 

152 "lxml.html.fromstring, defaulting to a per-thread parser. " 

153 "See local_deep_research.utilities.lxml_thread_safety." 

154 ) 

155 

156 lxml.html.fromstring = fromstring 

157 _installed = True 

158 logger.debug( 

159 "Installed per-thread lxml.html parsers " 

160 "(guards against concurrent xmlDict corruption)" 

161 ) 

162 return True 

163 

164 

165def is_installed() -> bool: 

166 """Whether the per-thread parser wrapper is currently active.""" 

167 return getattr(lxml.html.fromstring, "__wrapped__", None) is not None