Coverage for src/local_deep_research/journal_quality/data_sources/base.py: 62%

1"""DataSource base class for academic data downloads. 

2 

3Each external dataset (OpenAlex sources, DOAJ journals, Stop Predatory 

4Journals, JabRef abbreviations, …) is a subclass of `DataSource` that 

5declares its metadata as class attributes and implements the `fetch()` 

6method. The `data_sources` package's `ALL_SOURCES` registry then drives 

7the bulk download flow, the dashboard banner status endpoint, and the 

8lazy-load auto-download path in `JournalDataManager`. 

9""" 

10 

11from __future__ import annotations 

12 

13import os 

14from abc import ABC, abstractmethod 

15from pathlib import Path 

16from typing import Any 

17 

18from loguru import logger 

19 

20 

21def atomic_replace(tmp: Path, output: Path) -> None: 

22 """Move a fully-written temp file onto ``output``, overwriting it. 

23 

24 Uses ``os.replace`` rather than ``Path.rename``: on Windows ``rename`` 

25 raises ``FileExistsError`` if the destination already exists, which is 

26 the normal case when refreshing data that was downloaded before (POSIX 

27 ``rename`` overwrites silently, so the bug is invisible on Linux/macOS). 

28 ``os.replace`` overwrites the destination on both platforms and is atomic 

29 on POSIX (Windows can refuse it while a reader holds the file open - the 

30 same caveat tracked for the journal-quality DB in #6135). 

31 """ 

32 os.replace(tmp, output) 

33 

34 

35class DataSource(ABC): 

36 """Abstract base class for one downloadable academic data source. 

37 

38 Subclasses MUST override the metadata class attributes (key, name, 

39 url, license, license_url, description, filename, count_label) and 

40 implement `fetch()`. They MAY override `auto_download`, `required`, 

41 and `approx_size_mb`. 

42 """ 

43 

44 # ── metadata (subclasses MUST override) ──────────────────────── 

45 key: str = "" 

46 name: str = "" 

47 url: str = "" 

48 # Direct link to the actual dataset/dump that fetch() downloads — 

49 # the bulk file or manifest, not the project homepage. Surfaced 

50 # on the dashboard so users can inspect the upstream artifact. 

51 dataset_url: str = "" 

52 license: str = "" 

53 license_url: str = "" 

54 description: str = "" 

55 filename: str = "" 

56 count_label: str = "" 

57 

58 # ── policy (sensible defaults) ───────────────────────────────── 

59 # If True, lazy-load callers will fetch the file on demand. Set 

60 # only for small files (<1 MB) so first-use latency stays low. 

61 auto_download: bool = False 

62 # If True, a fetch() failure inside the bulk download loop is 

63 # fatal — the loop returns early with an error message. Other 

64 # sources are best-effort (logged and skipped). 

65 required: bool = False 

66 # Informational; surfaced to UIs that want to warn the user 

67 # about download size before they hit the button. 

68 approx_size_mb: float = 0.0 

69 

70 # ── methods (subclasses override only fetch) ─────────────────── 

71 

72 @abstractmethod 

73 def fetch(self, data_dir: Path, progress_cb=None) -> int: 

74 """Download from upstream and write `self.filename` into data_dir. 

75 

76 Subclasses are responsible for HTTP, parsing, and atomic write 

77 (download to a `.tmp` file then rename). Should raise on 

78 unrecoverable network/parse errors so the caller can decide 

79 whether to abort the bulk download or continue best-effort. 

80 

81 Args: 

82 data_dir: target directory for the compact output file. 

83 progress_cb: optional `Callable[[int, int, str], None]` 

84 that chunk-processing sources call periodically as 

85 ``progress_cb(done, total, detail)``. Used by the 

86 dashboard to drive a live per-source progress bar. 

87 One-shot sources (no loops) can ignore this. 

88 

89 Returns: 

90 Number of records fetched (used in success messages). 

91 """ 

92 

93 def is_present(self, data_dir: Path) -> bool: 

94 """True if the data file already exists in `data_dir`.""" 

95 return (data_dir / self.filename).exists() 

96 

97 def ensure(self, data_dir: Path) -> bool: 

98 """Ensure the data file is present, optionally fetching on demand. 

99 

100 - If already present → return True (no work). 

101 - If missing AND `auto_download=True` → fetch and return whether 

102 the file is present after the fetch attempt. 

103 - If missing AND `auto_download=False` → return False without 

104 touching the network. Caller is expected to drive the bulk 

105 download flow (dashboard "Download Data" button) instead. 

106 

107 Idempotent and safe to call repeatedly. Logs but does not 

108 re-raise exceptions from `fetch()`. 

109 """ 

110 if self.is_present(data_dir): 

111 return True 

112 if not self.auto_download: 

113 return False 

114 logger.info( 

115 f"{self.name} data not found — fetching from upstream " 

116 f"(~{self.approx_size_mb:.1f} MB)..." 

117 ) 

118 try: 

119 # Lazy import to avoid import cycles for this early-loaded 

120 # data-source registry module. 

121 from ...security.directory_creation import create_directory 

122 

123 create_directory( 

124 data_dir, context=f"journal quality data source '{self.key}'" 

125 ) 

126 self.fetch(data_dir) 

127 except Exception: 

128 logger.exception(f"Failed to fetch {self.name}") 

129 return False 

130 return self.is_present(data_dir) 

131 

132 def status_dict(self, data_dir: Path) -> dict[str, Any]: 

133 """Return the dict shape consumed by the dashboard banner JS. 

134 

135 Mirrors the entries the JS in `journal_quality.html` expects: 

136 `key`, `name`, `url`, `license`, `license_url`, `description`, 

137 `file`, `present`. 

138 """ 

139 return { 

140 "key": self.key, 

141 "name": self.name, 

142 "url": self.url, 

143 "dataset_url": self.dataset_url, 

144 "license": self.license, 

145 "license_url": self.license_url, 

146 "description": self.description, 

147 "file": self.filename, 

148 "present": self.is_present(data_dir), 

149 }