Coverage for src/local_deep_research/utilities/pandoc_conversion.py: 93%
122 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Bounded pandoc conversion for pandoc-backed uploads.
3Five upload formats convert through a ``pandoc`` subprocess before
4parsing: ``.rtf``/``.epub``/``.rst``/``.org`` through unstructured's
5``convert_file_to_html_text_using_pandoc`` and ``.odt`` through
6``partition_odt`` (odt -> docx), both ending in
7``pypandoc.convert_file`` -> ``Popen(...)`` + ``communicate()`` with
8**no timeout and no memory cap** — a hostile or pathological file pins
9an indexing worker indefinitely or grows pandoc's heap without limit.
11This module owns the conversion instead:
13- the binary is resolved the way pypandoc resolves it (the bundled
14 ``pypandoc-binary`` copy first), falling back to ``pandoc`` on PATH;
15- a wall-clock timeout enforced with a process-group kill (mirroring
16 the soffice wrapper in ``utilities/office_conversion.py``);
17- a memory cap on the pandoc child: a GHC RTS heap ceiling
18 (``+RTS -M<N>m -RTS``) when the binary accepts RTS options, otherwise
19 ``RLIMIT_AS`` in the child on Linux;
20- a ``--sandbox``ed argv with pandoc writing to an output file, whose
21 size is checked before it is read; pandoc's stdout/stderr go to
22 ``/dev/null`` so no diagnostic stream is buffered in the worker;
23- typed failures and full temp cleanup in ``finally``.
25Every extension converts straight to HTML, which the ``.html`` loader
26then parses.
27"""
29from __future__ import annotations
31import contextlib
32import functools
33import os
34import shutil
35import signal
36import subprocess
37import sys
38import tempfile
39import time
40from pathlib import Path
41from typing import Any, Callable, NamedTuple, Optional
43from loguru import logger
45try: # POSIX only; imported here so the forked child imports nothing.
46 import resource
47except ImportError: # pragma: no cover - Windows
48 resource = None # type: ignore[assignment]
50#: Every upload extension whose registered loader converts through
51#: pandoc; all of them are routed through this bounded wrapper instead.
52#: (.docx/.pptx/.xlsx are parsed in-process by python-docx/-pptx/
53#: openpyxl and never reach pandoc.)
54PANDOC_BACKED_EXTENSIONS = frozenset({".rtf", ".epub", ".rst", ".org", ".odt"})
56#: Conversion wall clock: real conversions of ordinary documents are
57#: sub-second to a few seconds; generous headroom keeps
58#: pathological-but-honest inputs from failing while a hung pandoc tree
59#: cannot outlive the worker's patience.
60DEFAULT_CONVERSION_TIMEOUT_S = 90
62#: Converted-HTML ceiling: extraction only needs text, and a larger
63#: "HTML" output signals amplification, not a real document.
64DEFAULT_MAX_OUTPUT_BYTES = 64 * 1024 * 1024
66#: pandoc heap ceiling in MiB. Measured with the bundled pandoc 3.9,
67#: peak RSS is roughly 100-120x the document's text size (1 MB of rst
68#: -> ~200 MB, 5 MB -> ~570 MB), and 20 MB of text needs ~2 GB *and*
69#: more than the 90 s timeout. 2 GiB therefore admits what the timeout
70#: admits, while a decompression bomb (pandoc's zip reader does not
71#: trust declared entry sizes) becomes a prompt, typed failure instead
72#: of an OOM of the host.
73DEFAULT_MAX_HEAP_MB = 2048
75#: Extra address space granted over the heap ceiling when the cap has
76#: to be enforced with RLIMIT_AS (code, RTS bookkeeping, mmaps).
77_RLIMIT_AS_HEADROOM_MB = 512
79#: GHC runtimes exit with this status on "Heap exhausted" (and on
80#: "out of memory" when RLIMIT_AS refuses an allocation).
81_GHC_HEAP_EXHAUSTED_EXIT = 251
83#: How long a *failed* RTS probe is remembered before it is retried.
84#: Successes are remembered for the life of the process; failures
85#: (including a timed-out probe on a loaded host) are retried, but at
86#: most once per window so a broken binary does not add a probe to
87#: every conversion.
88_RTS_PROBE_FAILURE_TTL_S = 300.0
90#: Timeout of one RTS probe (``pandoc +RTS -M64m -RTS --version``).
91_RTS_PROBE_TIMEOUT_S = 30
93#: binary path -> (probe succeeded, time.monotonic() of the probe)
94_rts_probe_cache: dict[str, tuple[bool, float]] = {}
97class PandocConversionError(ValueError):
98 """A pandoc conversion failed, timed out, or was refused."""
101def _kill_process_group(pid: int) -> None:
102 """Best-effort SIGKILL of *pid*'s whole process group.
104 On platforms without process groups (Windows) this falls back to
105 terminating *pid* alone: grandchildren pandoc spawned survive there.
106 """
107 with contextlib.suppress(ProcessLookupError, PermissionError):
108 if hasattr(os, "killpg"):
109 os.killpg(pid, signal.SIGKILL)
110 else: # pragma: no cover - Windows has no process groups
111 os.kill(pid, signal.SIGTERM)
114def _normalized_ext(extension: str) -> str:
115 return (
116 extension.lower()
117 if extension.startswith(".")
118 else f".{extension.lower()}"
119 )
122def is_pandoc_backed_extension(extension: str) -> bool:
123 """True when *extension*'s loader path converts through pandoc."""
124 return _normalized_ext(extension) in PANDOC_BACKED_EXTENSIONS
127def resolve_pandoc_binary() -> Optional[str]:
128 """Locate the pandoc binary the way the rest of the app does.
130 The shipped dependency is ``pypandoc-binary``, which bundles pandoc
131 inside the package (``pypandoc/files/pandoc``) and puts nothing on
132 PATH, so ``pypandoc.get_pandoc_path()`` is the primary resolver
133 (it also honours ``PYPANDOC_PANDOC`` and picks the newest of the
134 bundled and PATH copies). ``shutil.which`` is the fallback for
135 installs without pypandoc.
136 """
137 try:
138 import pypandoc
140 path = pypandoc.get_pandoc_path()
141 if path: 141 ↛ 145line 141 didn't jump to line 145 because the condition on line 141 was always true
142 return str(path)
143 except (ImportError, OSError, RuntimeError):
144 pass
145 return shutil.which("pandoc")
148def _probe_rts_heap_cap(pandoc: str) -> bool:
149 """Run one ``+RTS -M`` probe against *pandoc* (uncached)."""
150 try:
151 probe = subprocess.run( # noqa: S603 fixed argv, no shell
152 [pandoc, "+RTS", "-M64m", "-RTS", "--version"],
153 stdin=subprocess.DEVNULL,
154 stdout=subprocess.DEVNULL,
155 stderr=subprocess.DEVNULL,
156 timeout=_RTS_PROBE_TIMEOUT_S,
157 check=False,
158 )
159 except (OSError, subprocess.SubprocessError):
160 return False
161 return probe.returncode == 0
164def _pandoc_accepts_rts_heap_cap(pandoc: str) -> bool:
165 """Whether *pandoc* honours ``+RTS -M``.
167 Official pandoc builds (including the one pypandoc-binary bundles)
168 link with ``-rtsopts``; a build without it exits non-zero with
169 "Most RTS options are disabled". A success is cached for the
170 process; a failure only for ``_RTS_PROBE_FAILURE_TTL_S``, so a
171 transient failure (e.g. a probe timing out under load) does not
172 pin the process to the fallback cap for good.
173 """
174 now = time.monotonic()
175 cached = _rts_probe_cache.get(pandoc)
176 if cached is not None:
177 ok, probed_at = cached
178 if ok or now - probed_at < _RTS_PROBE_FAILURE_TTL_S:
179 return ok
180 ok = _probe_rts_heap_cap(pandoc)
181 _rts_probe_cache[pandoc] = (ok, now)
182 return ok
185def _set_address_space_limit(limit_bytes: int) -> None:
186 """Runs in the forked child before exec: ``setrlimit`` and nothing
187 else (``resource`` is imported by the parent at module load)."""
188 resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
191class _MemoryCap(NamedTuple):
192 rts_args: list[str]
193 preexec_fn: Optional[Callable[[], None]]
194 #: Human-readable effective limit, used in the failure message.
195 description: str
198def _memory_cap(pandoc: str, max_heap_mb: int) -> _MemoryCap:
199 """Return how pandoc's memory is capped for this binary.
201 Preferred: a GHC RTS heap ceiling on the argv — no Python code runs
202 in the forked child, so it is safe in this multi-threaded server.
203 Fallback for binaries without ``-rtsopts``: ``RLIMIT_AS`` of the
204 heap ceiling plus ``_RLIMIT_AS_HEADROOM_MB`` set in a
205 ``preexec_fn`` that does nothing else; Linux only, since macOS does
206 not enforce RLIMIT_AS and Windows has neither mechanism.
208 A probe that wrongly reports RTS support (the binary then rejects
209 ``+RTS``) costs availability only: the conversion fails typed and
210 pandoc never runs uncapped.
211 """
212 if _pandoc_accepts_rts_heap_cap(pandoc):
213 return _MemoryCap(
214 ["+RTS", f"-M{max_heap_mb}m", "-RTS"],
215 None,
216 f"{max_heap_mb} MiB heap cap",
217 )
218 if resource is not None and sys.platform.startswith("linux"): 218 ↛ 227line 218 didn't jump to line 227 because the condition on line 218 was always true
219 limit_mb = max_heap_mb + _RLIMIT_AS_HEADROOM_MB
220 return _MemoryCap(
221 [],
222 functools.partial(_set_address_space_limit, limit_mb * 1024 * 1024),
223 f"{limit_mb} MiB address-space limit "
224 f"({max_heap_mb} MiB heap + {_RLIMIT_AS_HEADROOM_MB} MiB "
225 "headroom)",
226 )
227 logger.warning(
228 "pandoc rejects RTS options and RLIMIT_AS is unavailable on this "
229 "platform; pandoc conversions are bounded by the timeout only"
230 )
231 return _MemoryCap([], None, "no memory cap")
234def pandoc_convert_bytes_to_html(
235 content: bytes,
236 extension: str,
237 *,
238 work_dir: Optional[Path] = None,
239 timeout_s: int = DEFAULT_CONVERSION_TIMEOUT_S,
240 max_output_bytes: int = DEFAULT_MAX_OUTPUT_BYTES,
241 max_heap_mb: int = DEFAULT_MAX_HEAP_MB,
242) -> bytes:
243 """Convert a pandoc-backed document to HTML bytes, bounded.
245 Returns the converted HTML. Raises ``PandocConversionError`` (a
246 ``ValueError``) on timeout, launch failure, memory-cap breach,
247 non-zero exit, missing output, or output over ``max_output_bytes``.
248 All temporary files are removed whether the conversion succeeds or
249 fails.
250 """
251 ext = _normalized_ext(extension)
252 if ext not in PANDOC_BACKED_EXTENSIONS: 252 ↛ 253line 252 didn't jump to line 253 because the condition on line 252 was never true
253 raise PandocConversionError(f"not a pandoc-backed extension: {ext}")
255 pandoc = resolve_pandoc_binary()
256 if pandoc is None:
257 raise PandocConversionError("pandoc binary not available")
259 cap = _memory_cap(pandoc, max_heap_mb)
261 owned_dir = work_dir is None
262 scratch = (
263 # nosemgrep: semgrep.rules.path-traversal-risk, reason: mkdtemp creates a private server-owned directory
264 Path(tempfile.mkdtemp(prefix="ldr-pandoc-"))
265 if owned_dir
266 # nosemgrep: semgrep.rules.path-traversal-risk, reason: upload caller omits this optional trusted-code path
267 else Path(work_dir).resolve()
268 )
269 in_path = scratch / f"in_doc{ext}"
270 out_path = scratch / "out.html"
272 try:
273 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Fixed filename and allowlisted extension inside a private conversion directory; upload callers cannot choose work_dir
274 in_path.write_bytes(content)
275 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Fixed out.html inside this call's conversion directory
276 out_path.unlink(missing_ok=True)
278 # --sandbox restricts pandoc's own file access (same posture
279 # as pypandoc's sandbox=True); output goes to a file so the
280 # size ceiling can be checked before anything is read, and
281 # --quiet plus /dev/null keeps warning floods out of memory.
282 command = [
283 pandoc,
284 *cap.rts_args,
285 "--sandbox",
286 "--quiet",
287 "--from",
288 ext.lstrip("."),
289 "--to",
290 "html",
291 "-o",
292 str(out_path),
293 str(in_path),
294 ]
295 popen_kwargs: dict[str, Any] = {}
296 if cap.preexec_fn is not None:
297 popen_kwargs["preexec_fn"] = cap.preexec_fn
298 try:
299 proc = subprocess.Popen( # noqa: S603 fixed argv, no shell
300 command,
301 stdin=subprocess.DEVNULL,
302 stdout=subprocess.DEVNULL,
303 stderr=subprocess.DEVNULL,
304 start_new_session=True,
305 **popen_kwargs,
306 )
307 try:
308 proc.communicate(timeout=timeout_s)
309 except subprocess.TimeoutExpired:
310 # start_new_session makes proc.pid the process-group
311 # id: kill the whole tree, then reap, then fail typed.
312 _kill_process_group(proc.pid)
313 proc.wait()
314 raise PandocConversionError(
315 f"pandoc conversion timed out after {timeout_s}s"
316 ) from None
317 except PandocConversionError:
318 raise
319 except (OSError, subprocess.SubprocessError) as exc:
320 raise PandocConversionError(
321 f"could not launch pandoc ({type(exc).__name__})"
322 ) from exc
324 if proc.returncode == _GHC_HEAP_EXHAUSTED_EXIT:
325 raise PandocConversionError(
326 f"pandoc conversion ran out of memory ({cap.description})"
327 )
328 if proc.returncode != 0:
329 raise PandocConversionError(
330 f"pandoc conversion failed (exit {proc.returncode})"
331 )
333 if not out_path.exists():
334 raise PandocConversionError("pandoc conversion produced no output")
336 output_size = out_path.stat().st_size
337 if output_size > max_output_bytes:
338 raise PandocConversionError(
339 f"converted output is larger than {max_output_bytes} bytes"
340 )
342 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Read the size-checked fixed out.html from this call's conversion directory
343 converted = out_path.read_bytes()
344 logger.debug(
345 "pandoc conversion ok: {} -> html ({} bytes)",
346 ext,
347 output_size,
348 )
349 return converted
350 finally:
351 for path in (in_path, out_path):
352 try:
353 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Cleanup is limited to this conversion's fixed input and output filenames
354 path.unlink(missing_ok=True)
355 except OSError:
356 logger.warning("could not remove conversion temp file")
357 if owned_dir:
358 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Only this call's mkdtemp directory is recursively removed; caller-owned work directories are retained
359 shutil.rmtree(scratch, ignore_errors=True)