Coverage for src/local_deep_research/journal_quality/data_sources/predatory.py: 89%

1"""Stop Predatory Journals data source. 

2 

3Community successor to Jeffrey Beall's original predatory publishers 

4list (Beall took down his original blog post in 2017). The successor 

5project maintains three CSV files (publishers, journals, hijacked) on 

6GitHub which we merge into a single predatory.json. 

7""" 

8 

9from __future__ import annotations 

10 

11import csv 

12import io 

13import json 

14from pathlib import Path 

15 

16from loguru import logger 

17 

18from .base import DataSource, atomic_replace 

19 

20_PREDATORY_BASE = ( 

21 "https://raw.githubusercontent.com/stop-predatory-journals/" 

22 "stop-predatory-journals.github.io/master/_data" 

23) 

24_PREDATORY_FILES = { 

25 "publishers": "publishers.csv", 

26 "journals": "journals.csv", 

27 "hijacked": "hijacked.csv", 

28} 

29 

30# Safety floor — the upstream lists carry thousands of entries each. 

31# Refuse to overwrite a healthy on-disk snapshot with a near-empty 

32# payload (e.g. CDN partial outage where two of the three CSVs 

33# returned 0 rows) since that would silently disable predatory 

34# filtering for everyone. 

35_MIN_PREDATORY_TOTAL = 100 

36 

37 

38class PredatorySource(DataSource): 

39 key = "predatory" # gitleaks:allow 

40 name = "Stop Predatory Journals" 

41 url = ( 

42 "https://github.com/stop-predatory-journals/" 

43 "stop-predatory-journals.github.io" 

44 ) 

45 dataset_url = ( 

46 "https://github.com/stop-predatory-journals/" 

47 "stop-predatory-journals.github.io/tree/master/_data" 

48 ) 

49 license = "MIT" 

50 license_url = "https://opensource.org/license/mit" 

51 description = ( 

52 "Community successor to Beall's List — predatory publishers, " 

53 "journals, and hijacked journal entries" 

54 ) 

55 filename = "predatory.json" 

56 count_label = "predatory entries" 

57 auto_download = True # ~0.3 MB; fetch on first filter use 

58 required = False 

59 approx_size_mb = 0.3 

60 

61 def fetch(self, data_dir: Path, progress_cb=None) -> int: 

62 from ...security.safe_requests import ( 

63 safe_get_with_retries as safe_get, 

64 ) 

65 

66 publishers: list[dict] = [] 

67 journals: list[dict] = [] 

68 hijacked: list[dict] = [] 

69 

70 def _read_csv(filename: str) -> list[dict]: 

71 url = f"{_PREDATORY_BASE}/{filename}" 

72 resp = safe_get( 

73 url, timeout=30, consume_body=True, require_https=True 

74 ) 

75 resp.raise_for_status() 

76 reader = csv.DictReader(io.StringIO(resp.text)) 

77 return [ 

78 {k: (v or "").strip() for k, v in row.items() if k} 

79 for row in reader 

80 ] 

81 

82 for row in _read_csv(_PREDATORY_FILES["publishers"]): 

83 name = row.get("name", "") 

84 if name: 84 ↛ 82line 84 didn't jump to line 82 because the condition on line 84 was always true

85 publishers.append({"name": name, "url": row.get("url", "")}) 

86 

87 for row in _read_csv(_PREDATORY_FILES["journals"]): 

88 name = row.get("name", "") 

89 if name: 89 ↛ 87line 89 didn't jump to line 87 because the condition on line 89 was always true

90 journals.append({"name": name, "url": row.get("url", "")}) 

91 

92 for row in _read_csv(_PREDATORY_FILES["hijacked"]): 92 ↛ 96line 92 didn't jump to line 96 because the loop on line 92 never started

93 # Upstream column names: hijacked, hijackedabbr, hijackedurl, 

94 # althijackedurl, authentic, authenticabbr, authenticurl. 

95 # The rest of the codebase reads `hijacked_name`, so map across. 

96 name = row.get("hijacked", "") 

97 if name: 

98 hijacked.append( 

99 { 

100 "hijacked_name": name, 

101 "original_name": row.get("authentic", ""), 

102 "hijacked_url": row.get("hijackedurl", ""), 

103 "original_url": row.get("authenticurl", ""), 

104 } 

105 ) 

106 

107 payload = { 

108 "metadata": { 

109 "source": ( 

110 "Stop Predatory Journals " 

111 "(https://github.com/stop-predatory-journals/" 

112 "stop-predatory-journals.github.io) — " 

113 "community successor to Beall's List" 

114 ), 

115 "license": "MIT", 

116 "publisher_count": len(publishers), 

117 "journal_count": len(journals), 

118 "hijacked_count": len(hijacked), 

119 }, 

120 "publishers": publishers, 

121 "journals": journals, 

122 "hijacked": hijacked, 

123 } 

124 

125 total = len(publishers) + len(journals) + len(hijacked) 

126 if total < _MIN_PREDATORY_TOTAL: 

127 raise RuntimeError( 

128 f"Predatory: suspiciously few records " 

129 f"({total} < {_MIN_PREDATORY_TOTAL}); refusing to " 

130 "overwrite existing data" 

131 ) 

132 

133 output = data_dir / self.filename 

134 tmp = data_dir / f"{self.filename}.tmp" 

135 with open(tmp, "w", encoding="utf-8") as f: 

136 json.dump(payload, f) 

137 atomic_replace(tmp, output) 

138 

139 logger.info( 

140 f"Predatory: saved {len(publishers)} publishers + " 

141 f"{len(journals)} journals + {len(hijacked)} hijacked" 

142 ) 

143 return total