Coverage for src/local_deep_research/document_loaders/xls_loader.py: 92%

1"""Custom loader for legacy ``.xls`` (BIFF) spreadsheets. 

2 

3``unstructured``'s Excel partitioner runs a ``msoffcrypto`` "is encrypted?" 

4pre-check on every workbook. That check crashes on some legacy ``.xls`` files 

5(``struct.error: unpack requires a buffer of 4 bytes`` in its xls97 record 

6reader) even though ``xlrd`` reads the same file fine. To make ``.xls`` uploads 

7extract reliably, this loader reads the workbook directly with pandas + xlrd 

8and bypasses that pre-check. Modern ``.xlsx`` files still use the unstructured 

9loader, whose OOXML path is unaffected. 

10""" 

11 

12from pathlib import Path 

13from typing import Iterator 

14 

15import pandas as pd 

16from langchain_core.document_loaders import BaseLoader 

17from langchain_core.documents import Document 

18from loguru import logger 

19 

20 

21class XLSLoader(BaseLoader): 

22 """Load a legacy ``.xls`` workbook and convert each sheet to text. 

23 

24 Every sheet is read with the ``xlrd`` engine and rendered as one 

25 space-joined line per row (skipping empty cells), which mirrors the plain 

26 text that the unstructured Excel loader produces for ``.xlsx``. 

27 """ 

28 

29 def __init__(self, file_path: str | Path, **kwargs: object): 

30 self.file_path = Path(file_path) 

31 

32 def lazy_load(self) -> Iterator[Document]: 

33 try: 

34 # sheet_name=None reads all sheets; header=None keeps the first row 

35 # as data so column headers are included in the extracted text. 

36 sheets = pd.read_excel( 

37 self.file_path, 

38 sheet_name=None, 

39 header=None, 

40 engine="xlrd", 

41 # pandas reads the literal strings "N/A", "NA", "n/a", "NULL", 

42 # "None", "NaN" and "nan" as missing values, so a cell saying any 

43 # of them would be dropped by the pd.notna filter below. "N/A" is 

44 # how a person writes "not applicable"; the cell means something. 

45 keep_default_na=False, 

46 ) 

47 except Exception as exc: 

48 msg = str(exc).lower() 

49 if "encrypt" in msg or "password" in msg: 

50 raise ValueError( 

51 f"XLS file is encrypted and cannot be read: {self.file_path}" 

52 ) from exc 

53 logger.exception(f"Error loading XLS file: {self.file_path}") 

54 raise 

55 

56 for sheet_name, frame in sheets.items(): 

57 lines = [] 

58 for row in frame.itertuples(index=False, name=None): 

59 # A blank cell now arrives as "" rather than NaN, and is 

60 # skipped the same way. 

61 cells = [ 

62 text 

63 for value in row 

64 if pd.notna(value) and (text := str(value)) != "" 

65 ] 

66 if cells: 66 ↛ 58line 66 didn't jump to line 58 because the condition on line 66 was always true

67 lines.append(" ".join(cells)) 

68 text = "\n".join(lines).strip() 

69 if not text: 69 ↛ 70line 69 didn't jump to line 70 because the condition on line 69 was never true

70 continue 

71 yield Document( 

72 page_content=text, 

73 metadata={ 

74 "source": str(self.file_path), 

75 "file_type": "xls", 

76 "sheet_name": sheet_name, 

77 }, 

78 ) 

79 

80 def load(self) -> list[Document]: 

81 """Load all sheets from the XLS file.""" 

82 return list(self.lazy_load())