Coverage for src/local_deep_research/document_loaders/xls_loader.py: 92%

1"""Custom loader for legacy ``.xls`` (BIFF) spreadsheets. 

2 

3``unstructured``'s Excel partitioner runs a ``msoffcrypto`` "is encrypted?" 

4pre-check on every workbook. That check crashes on some legacy ``.xls`` files 

5(``struct.error: unpack requires a buffer of 4 bytes`` in its xls97 record 

6reader) even though ``xlrd`` reads the same file fine. To make ``.xls`` uploads 

7extract reliably, this loader reads the workbook directly with pandas + xlrd 

8and bypasses that pre-check. Modern ``.xlsx`` files still use the unstructured 

9loader, whose OOXML path is unaffected. 

10""" 

11 

12from pathlib import Path 

13from typing import Iterator 

14 

15import pandas as pd 

16from langchain_core.document_loaders import BaseLoader 

17from langchain_core.documents import Document 

18from loguru import logger 

19 

20 

21class XLSLoader(BaseLoader): 

22 """Load a legacy ``.xls`` workbook and convert each sheet to text. 

23 

24 Every sheet is read with the ``xlrd`` engine and rendered as one 

25 space-joined line per row (skipping empty cells), which mirrors the plain 

26 text that the unstructured Excel loader produces for ``.xlsx``. 

27 """ 

28 

29 def __init__(self, file_path: str | Path, **kwargs: object): 

30 self.file_path = Path(file_path) 

31 

32 def lazy_load(self) -> Iterator[Document]: 

33 try: 

34 # sheet_name=None reads all sheets; header=None keeps the first row 

35 # as data so column headers are included in the extracted text. 

36 sheets = pd.read_excel( 

37 self.file_path, 

38 sheet_name=None, 

39 header=None, 

40 engine="xlrd", 

41 ) 

42 except Exception as exc: 

43 msg = str(exc).lower() 

44 if "encrypt" in msg or "password" in msg: 

45 raise ValueError( 

46 f"XLS file is encrypted and cannot be read: {self.file_path}" 

47 ) from exc 

48 logger.exception(f"Error loading XLS file: {self.file_path}") 

49 raise 

50 

51 for sheet_name, frame in sheets.items(): 

52 lines = [] 

53 for row in frame.itertuples(index=False, name=None): 

54 cells = [str(value) for value in row if pd.notna(value)] 

55 if cells: 55 ↛ 53line 55 didn't jump to line 53 because the condition on line 55 was always true

56 lines.append(" ".join(cells)) 

57 text = "\n".join(lines).strip() 

58 if not text: 58 ↛ 59line 58 didn't jump to line 59 because the condition on line 58 was never true

59 continue 

60 yield Document( 

61 page_content=text, 

62 metadata={ 

63 "source": str(self.file_path), 

64 "file_type": "xls", 

65 "sheet_name": sheet_name, 

66 }, 

67 ) 

68 

69 def load(self) -> list[Document]: 

70 """Load all sheets from the XLS file.""" 

71 return list(self.lazy_load())