Coverage for src/local_deep_research/utilities/office_conversion.py: 90%
80 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Bounded conversion of legacy office formats (.doc/.ppt).
3``.doc``/``.ppt`` uploads are OLE binaries; the loader path converts
4them to modern containers via a ``soffice`` subprocess before parsing.
5The upstream conversion (unstructured's ``convert_office_doc``) invokes
6``soffice`` with **no timeout**, so a hostile or pathological file can
7pin an indexing worker forever — for content up to the 3 GB upload cap.
9This module owns the conversion instead: only input carrying the OLE
10compound-file signature is handed to soffice, a real wall-clock timeout
11enforced with a process-group kill (``soffice`` spawns children that a
12bare ``subprocess.run(timeout=...)`` would leave alive), typed
13failures, output validation (expected file, size ceiling), soffice's
14stdio sent to ``/dev/null`` rather than buffered, a per-call
15``UserInstallation`` profile so concurrent conversions cannot collide
16on soffice's singleton profile lock, and full temp cleanup in
17``finally``. soffice's memory is not capped (it is bounded by the
18timeout only); the OLE-signature check keeps renamed zip containers
19and other formats from reaching it.
20"""
22from __future__ import annotations
24import contextlib
25import os
26import shutil
27import signal
28import subprocess
29import tempfile
30import uuid
31from pathlib import Path
32from typing import Optional
34from loguru import logger
36#: Legacy extensions -> the modern container soffice produces.
37_LEGACY_TARGETS = {".doc": ".docx", ".ppt": ".pptx"}
39#: OLE compound-file signature: every genuine .doc/.ppt starts with it.
40OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
42#: soffice binary candidates, resolved once per call via shutil.which.
43_SOFFICE_CANDIDATES = ("soffice", "libreoffice")
45#: Conversion wall clock. Generous for real documents (a 3 GB file is
46#: rejected by the upload cap long before this; honest conversions of
47#: large-but-legitimate documents finish in tens of seconds) while
48#: keeping a hung process tree from pinning a worker forever.
49DEFAULT_CONVERSION_TIMEOUT_S = 120
51#: Converted-output ceiling. A conversion that legitimately needs more
52#: than this was never going to be indexed usefully.
53DEFAULT_MAX_OUTPUT_BYTES = 512 * 1024 * 1024
56class OfficeConversionError(ValueError):
57 """A legacy-office conversion failed, timed out, or was refused."""
60def _normalized_ext(extension: str) -> str:
61 return (
62 extension.lower()
63 if extension.startswith(".")
64 else f".{extension.lower()}"
65 )
68def _kill_process_group(pid: int) -> None:
69 """Best-effort SIGKILL of *pid*'s whole process group.
71 On platforms without process groups (Windows) this falls back to
72 terminating *pid* alone: soffice's helper processes survive there.
73 """
74 with contextlib.suppress(ProcessLookupError, PermissionError):
75 if hasattr(os, "killpg"):
76 os.killpg(pid, signal.SIGKILL)
77 else: # pragma: no cover - Windows has no process groups
78 os.kill(pid, signal.SIGTERM)
81def convert_legacy_office_bytes(
82 content: bytes,
83 extension: str,
84 *,
85 output_dir: Optional[Path] = None,
86 timeout_s: int = DEFAULT_CONVERSION_TIMEOUT_S,
87 max_output_bytes: int = DEFAULT_MAX_OUTPUT_BYTES,
88 input_stem: Optional[str] = None,
89) -> bytes:
90 """Convert legacy OLE office bytes to their modern container.
92 Returns the converted container's bytes. Raises
93 ``OfficeConversionError`` (a ``ValueError``) on timeout, non-zero
94 exit, missing/unexpected output, or an output over
95 ``max_output_bytes``. All temporary files are removed whether the
96 conversion succeeds or fails.
97 """
98 ext = _normalized_ext(extension)
99 target = _LEGACY_TARGETS.get(ext)
100 if target is None: 100 ↛ 101line 100 didn't jump to line 101 because the condition on line 100 was never true
101 raise OfficeConversionError(f"not a legacy office extension: {ext}")
103 # soffice sniffs content, not the extension: anything else renamed
104 # .doc/.ppt (a zip bomb, an RTF, an HTML page) would reach a filter
105 # outside every other guard, with no memory cap on soffice. Only
106 # the OLE compound files these extensions denote are converted.
107 if not content.startswith(OLE_MAGIC):
108 raise OfficeConversionError(f"{ext} upload is not an OLE compound file")
110 soffice = next(
111 (name for name in _SOFFICE_CANDIDATES if shutil.which(name)), None
112 )
113 if soffice is None: 113 ↛ 114line 113 didn't jump to line 114 because the condition on line 113 was never true
114 raise OfficeConversionError("soffice binary not available")
116 # mkdtemp is absolute by construction; a caller-supplied output_dir
117 # may be relative, which would break profile_dir.as_uri() below.
118 work_dir = (
119 # nosemgrep: semgrep.rules.path-traversal-risk, reason: mkdtemp creates a private server-owned directory
120 Path(tempfile.mkdtemp(prefix="ldr-office-"))
121 if output_dir is None
122 # nosemgrep: semgrep.rules.path-traversal-risk, reason: upload caller omits this optional trusted-code path
123 else Path(output_dir).resolve()
124 )
126 if input_stem is None:
127 # nosemgrep: semgrep.rules.sql-string-concatenation, reason: validated local filename stem; no SQL sink
128 stem = f"in_{uuid.uuid4().hex}"
129 elif input_stem and all(c.isalnum() or c in "-_" for c in input_stem):
130 stem = input_stem
131 else:
132 # A traversal-shaped stem must never escape the work dir.
133 raise OfficeConversionError("invalid conversion input stem")
135 in_path = work_dir / f"{stem}{ext}"
136 out_path = work_dir / f"{stem}{target}"
137 # Per-call profile: soffice serializes on a shared UserInstallation
138 # and concurrent conversions would block or corrupt each other.
139 profile_dir = work_dir / f"profile_{uuid.uuid4().hex}"
141 try:
142 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Validated stem and allowlisted extension inside a private conversion directory; upload callers cannot choose output_dir
143 in_path.write_bytes(content)
144 # Defensive pre-clean: a stale file at out_path from an earlier
145 # call in a shared output_dir must not fake a successful run.
146 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Fixed converted filename derived from the validated stem and allowlisted target extension in the conversion directory
147 out_path.unlink(missing_ok=True)
149 command = [
150 soffice,
151 "--headless",
152 "--norestore",
153 "--nolockcheck",
154 f"-env:UserInstallation={profile_dir.as_uri()}",
155 "--convert-to",
156 target.lstrip("."),
157 "--outdir",
158 str(work_dir),
159 str(in_path),
160 ]
161 try:
162 proc = subprocess.Popen( # noqa: S603 fixed argv, no shell
163 command,
164 # Nothing is read from soffice's streams: /dev/null
165 # keeps a chatty or hostile conversion from growing
166 # an in-memory pipe buffer in the worker.
167 stdin=subprocess.DEVNULL,
168 stdout=subprocess.DEVNULL,
169 stderr=subprocess.DEVNULL,
170 start_new_session=True,
171 )
172 try:
173 proc.communicate(timeout=timeout_s)
174 except subprocess.TimeoutExpired:
175 # start_new_session makes proc.pid the process-group
176 # id: kill the whole tree, then reap, then fail typed.
177 _kill_process_group(proc.pid)
178 proc.wait()
179 raise OfficeConversionError(
180 f"legacy office conversion timed out after {timeout_s}s"
181 ) from None
182 except OfficeConversionError:
183 raise
184 except OSError as exc:
185 raise OfficeConversionError(
186 f"could not launch soffice ({type(exc).__name__})"
187 ) from exc
189 if proc.returncode != 0: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 raise OfficeConversionError(
191 f"legacy office conversion failed (exit {proc.returncode})"
192 )
194 if not out_path.exists():
195 raise OfficeConversionError(
196 "legacy office conversion produced no output"
197 )
199 output_size = out_path.stat().st_size
200 if output_size > max_output_bytes:
201 raise OfficeConversionError(
202 f"converted output is larger than {max_output_bytes} bytes"
203 )
205 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Read the size-checked converter output with the fixed target extension from this call's conversion directory
206 converted = out_path.read_bytes()
207 logger.debug(
208 "legacy office conversion ok: {} -> {} ({} bytes)",
209 ext,
210 target,
211 output_size,
212 )
213 return converted
214 finally:
215 # Full temp hygiene on every path, success or failure: the
216 # caller's content, soffice's output, and the profile tree.
217 for path in (in_path, out_path):
218 try:
219 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Cleanup is limited to this conversion's validated input and output filenames
220 path.unlink(missing_ok=True)
221 except OSError:
222 logger.warning("could not remove conversion temp file")
223 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Profile directory uses a fresh UUID beneath this call's conversion directory
224 shutil.rmtree(profile_dir, ignore_errors=True)
225 if output_dir is None:
226 # nosemgrep: semgrep.rules.path-traversal-risk, reason: Only this call's mkdtemp directory is recursively removed; caller-owned output directories are retained
227 shutil.rmtree(work_dir, ignore_errors=True)
230def is_legacy_office_extension(extension: str) -> bool:
231 """True when *extension* is one of the OLE formats needing conversion."""
232 return _normalized_ext(extension) in _LEGACY_TARGETS
235def converted_extension(extension: str) -> str:
236 """The modern container extension a legacy *extension* converts to."""
237 return _LEGACY_TARGETS[_normalized_ext(extension)]