Coverage for src/local_deep_research/document_loaders/bytes_loader.py: 92%
80 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Load documents from bytes content.
4This module provides functions to load documents from in-memory bytes,
5which is useful for handling file uploads via HTTP.
6"""
8import tempfile
9from pathlib import Path
10from typing import Optional
12from langchain_core.documents import Document
13from loguru import logger
15from local_deep_research.security.filename_sanitizer import sanitize_filename
17from .openpyxl_hardening import disable_openpyxl_drawing_reads
18from .zip_container_guard import (
19 DecompressionBombError,
20 validate_zip_container,
21)
23from .loader_registry import (
24 get_loader_class_for_extension,
25 is_extension_supported,
26)
27from ..utilities.office_conversion import (
28 OfficeConversionError,
29 convert_legacy_office_bytes,
30 converted_extension,
31 is_legacy_office_extension,
32)
33from ..utilities.pandoc_conversion import (
34 PandocConversionError,
35 is_pandoc_backed_extension,
36 pandoc_convert_bytes_to_html,
37)
40def load_from_bytes(
41 content: bytes,
42 extension: str,
43 filename: str = "upload",
44 source_url: Optional[str] = None,
45) -> list[Document]:
46 """
47 Load documents from bytes content.
49 PDFs are extracted directly from memory with shared page, character,
50 and CPU ceilings. Other formats are written to a temporary file, loaded
51 with the registered LangChain loader, then cleaned up.
53 Args:
54 content: File content as bytes
55 extension: File extension (with or without leading dot)
56 filename: Original filename for metadata
57 source_url: Optional source URL for metadata
59 Returns:
60 List of Document objects with extracted content
62 Raises:
63 ValueError: If the extension is not supported
64 """
65 # Defense in depth: re-sanitize filename even though callers
66 # should have sanitized already
67 try:
68 filename = sanitize_filename(filename)
69 except Exception:
70 filename = "upload"
72 # Normalize extension
73 ext = (
74 extension.lower()
75 if extension.startswith(".")
76 else f".{extension.lower()}"
77 )
79 # Check if extension is supported
80 if not is_extension_supported(ext):
81 raise ValueError(f"Unsupported file extension: {ext}")
83 # Legacy OLE binaries (.doc/.ppt) convert through a bounded soffice
84 # subprocess FIRST: the raw bytes are not zip containers, and the
85 # converted output then flows through the same zip guard as any
86 # other container below. The converter refuses input without the
87 # OLE signature, so a zip renamed .doc/.ppt never reaches soffice.
88 if is_legacy_office_extension(ext):
89 content = convert_legacy_office_bytes(content, ext)
90 ext = converted_extension(ext)
92 # Zip-container documents (.docx/.xlsx/.odt) decompress inside the
93 # parsers, whose reads are bounded only by the archive's declared
94 # entry sizes — the request-body cap never sees the expanded side.
95 # Refuse hostile declarations before any parser runs. This bounds
96 # archive reads and specific fan-outs, not parser memory or time in
97 # general (see "Known gaps" in zip_container_guard). This runs
98 # BEFORE the pandoc conversion below so an .epub container is
99 # guarded as the original archive.
100 validate_zip_container(content, ext)
102 # openpyxl re-reads an image once per drawing reference and keeps
103 # every copy, which no per-entry ceiling bounds; drawings carry no
104 # text, so the .xlsx path never reads them.
105 if ext == ".xlsx" and not disable_openpyxl_drawing_reads():
106 raise DecompressionBombError(
107 "Cannot safely read .xlsx: openpyxl drawing guard unavailable"
108 )
110 # Pandoc-backed formats (.rtf/.epub/.rst/.org/.odt) convert to HTML
111 # through a bounded subprocess (timeout, heap cap, output ceiling)
112 # before the HTML loader parses them; their registered unstructured
113 # loaders would otherwise run pandoc with no bounds at all.
114 if is_pandoc_backed_extension(ext):
115 content = pandoc_convert_bytes_to_html(content, ext)
116 ext = ".html"
118 # Get the loader class for this extension
119 loader_info = get_loader_class_for_extension(ext)
120 if loader_info is None:
121 raise ValueError(f"No loader found for extension: {ext}")
123 loader_class, loader_kwargs = loader_info
125 tmp_path = None
126 try:
127 # PDF text must use the same page/character/CPU ceilings as the
128 # library download and research-upload extractors. Both RAG uploads
129 # and Zotero sync enter through this byte-backed loader.
130 if ext == ".pdf":
131 from .bounded_pdf import load_bounded_pdf
133 documents = load_bounded_pdf(content, filename)
134 else:
135 with tempfile.NamedTemporaryFile(
136 suffix=ext, delete=False, prefix="ldr_upload_"
137 ) as tmp:
138 tmp.write(content)
139 tmp_path = tmp.name
140 loader = loader_class(tmp_path, **loader_kwargs)
141 documents = loader.load()
143 # Add metadata to all documents
144 for doc in documents:
145 doc.metadata["original_filename"] = filename
146 if source_url:
147 doc.metadata["source_url"] = source_url
149 logger.info(
150 f"Loaded {len(documents)} document(s) from {filename} ({ext})"
151 )
152 return documents
154 except (ImportError, ModuleNotFoundError):
155 # The extension is registered but the loader's runtime parser
156 # dependency is missing. The registry normally prevents this
157 # (see loader_registry capability flags), so surface it loudly
158 # rather than as a generic extraction failure.
159 logger.exception(
160 f"Missing optional dependency for {ext} files while loading "
161 f"{filename}; the loader for this format is not fully installed"
162 )
163 raise
165 except Exception:
166 logger.exception(f"Error loading {filename} ({ext})")
167 raise
169 finally:
170 # Clean up temp file
171 if tmp_path:
172 try:
173 # nosemgrep: semgrep.rules.path-traversal-risk, reason: tmp_path is this call's NamedTemporaryFile name; the upload filename is not used as a path
174 Path(tmp_path).unlink(missing_ok=True)
175 except Exception:
176 logger.warning(f"Failed to clean up temp file: {tmp_path}")
179def extract_text_from_bytes(
180 content: bytes,
181 extension: str,
182 filename: str = "upload",
183) -> Optional[str]:
184 """
185 Extract text from bytes content.
187 This is a convenience function that loads documents and joins their content.
189 Args:
190 content: File content as bytes
191 extension: File extension (with or without leading dot)
192 filename: Original filename for logging
194 Returns:
195 Extracted text as string, or None if extraction failed
196 """
197 try:
198 documents = load_from_bytes(content, extension, filename)
199 if documents:
200 return "\n\n".join(
201 doc.page_content for doc in documents if doc.page_content
202 )
203 return None
204 except DecompressionBombError:
205 # Every refusal of the zip-container guard lands here, ordinary
206 # documents over its size, structure or parsing-work limits as
207 # well as hostile ones, so the log names no cause; it is logged
208 # without the user-controlled filename, entry names or the
209 # exception's message (which can carry an entry name), and
210 # surfaces to the upload route as a per-file error entry.
211 logger.warning(
212 "Refused document: it exceeds the parsing limits for its "
213 "format (size, structure or estimated parsing work)"
214 )
215 return None
216 except OfficeConversionError:
217 # Timed-out, failed, or oversized legacy conversion: scrubbed
218 # message only, no user-controlled content.
219 logger.warning(
220 "Rejected legacy office document: conversion failed, timed "
221 "out, or produced an oversized result"
222 )
223 return None
224 except PandocConversionError:
225 logger.warning(
226 "Rejected pandoc-backed document: conversion failed, timed "
227 "out, or produced an oversized result"
228 )
229 return None
230 except ValueError:
231 logger.warning(
232 f"No text extracted from {filename}: unsupported format or "
233 "unrecognized text encoding"
234 )
235 return None
236 except Exception:
237 logger.exception(f"Error extracting text from {filename}")
238 return None