Coverage for src/local_deep_research/journal_quality/data_sources/jabref.py: 86%

1"""JabRef journal abbreviations data source. 

2 

3Downloads ~14 small CSV files from the JabRef GitHub repo and merges 

4them into a single gzipped JSON mapping abbreviations → full names. 

5""" 

6 

7from __future__ import annotations 

8 

9import csv 

10import gzip 

11import io 

12import json 

13from pathlib import Path 

14 

15from loguru import logger 

16 

17from .base import DataSource, atomic_replace 

18 

19_JABREF_BASE = ( 

20 "https://raw.githubusercontent.com/JabRef/abbrv.jabref.org/main/journals" 

21) 

22_JABREF_FILES = [ 

23 "journal_abbreviations_entrez.csv", 

24 "journal_abbreviations_ubc.csv", 

25 "journal_abbreviations_lifescience.csv", 

26 "journal_abbreviations_mechanical.csv", 

27 "journal_abbreviations_mathematics.csv", 

28 "journal_abbreviations_medicus.csv", 

29 "journal_abbreviations_ams.csv", 

30 "journal_abbreviations_general.csv", 

31 "journal_abbreviations_acs.csv", 

32 "journal_abbreviations_geology_physics.csv", 

33 "journal_abbreviations_ieee.csv", 

34 "journal_abbreviations_meteorology.csv", 

35 "journal_abbreviations_astronomy.csv", 

36 "journal_abbreviations_sociology.csv", 

37] 

38 

39 

40class JabRefSource(DataSource): 

41 key = "jabref" # gitleaks:allow 

42 name = "JabRef abbreviations" 

43 url = "https://github.com/JabRef/abbrv.jabref.org" 

44 dataset_url = ( 

45 "https://github.com/JabRef/abbrv.jabref.org/tree/main/journals" 

46 ) 

47 license = "CC0 1.0" 

48 license_url = "https://creativecommons.org/publicdomain/zero/1.0/" 

49 description = "~66K journal abbreviation → full name mappings" 

50 filename = "jabref_abbreviations.json.gz" 

51 count_label = "abbreviations" 

52 auto_download = True # ~0.5 MB total across 14 files 

53 required = False 

54 approx_size_mb = 0.5 

55 

56 def fetch(self, data_dir: Path, progress_cb=None) -> int: 

57 from ...security.safe_requests import ( 

58 safe_get_with_retries as safe_get, 

59 ) 

60 

61 abbrev_to_full: dict[str, str] = {} 

62 

63 for filename in _JABREF_FILES: 

64 url = f"{_JABREF_BASE}/{filename}" 

65 try: 

66 resp = safe_get( 

67 url, timeout=30, consume_body=True, require_https=True 

68 ) 

69 resp.raise_for_status() 

70 reader = csv.reader(io.StringIO(resp.text)) 

71 for row in reader: 

72 if len(row) < 2: 72 ↛ 73line 72 didn't jump to line 73 because the condition on line 72 was never true

73 continue 

74 full_name = row[0].strip().strip('"') 

75 abbreviation = row[1].strip().strip('"') 

76 if full_name and abbreviation and full_name != abbreviation: 76 ↛ 71line 76 didn't jump to line 71 because the condition on line 76 was always true

77 abbrev_lower = abbreviation.lower() 

78 # Last-writer-wins across 14 source CSVs. Log the 

79 # collision at debug level (one per file change, 

80 # not per row) so operators can audit which 

81 # source resolves a given abbreviation — not a 

82 # warning, because collisions are expected. 

83 if ( 83 ↛ 87line 83 didn't jump to line 87 because the condition on line 83 was never true

84 abbrev_lower in abbrev_to_full 

85 and abbrev_to_full[abbrev_lower] != full_name 

86 ): 

87 logger.debug( 

88 "jabref collision " 

89 f"[{filename}] {abbrev_lower!r}: " 

90 f"{abbrev_to_full[abbrev_lower]!r} → " 

91 f"{full_name!r}" 

92 ) 

93 abbrev_to_full[abbrev_lower] = full_name 

94 # Also store without dots: "Phys Rev Lett" → same 

95 no_dots = abbreviation.replace(".", "").strip().lower() 

96 if no_dots != abbreviation.lower(): 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true

97 if ( 

98 no_dots in abbrev_to_full 

99 and abbrev_to_full[no_dots] != full_name 

100 ): 

101 logger.debug( 

102 "jabref collision (no-dots) " 

103 f"[{filename}] {no_dots!r}: " 

104 f"{abbrev_to_full[no_dots]!r} → " 

105 f"{full_name!r}" 

106 ) 

107 abbrev_to_full[no_dots] = full_name 

108 except Exception: 

109 # Preserve traceback — operators diagnosing partial 

110 # fetch failures need the exception type (timeout, 

111 # SSRF block, decode error, etc.), not just the 

112 # filename. The outer loop tolerates per-file 

113 # failures, so this is a non-fatal warning. 

114 logger.exception(f"Failed to fetch JabRef file: {filename}") 

115 continue 

116 

117 # Sanity check: if every single file failed (e.g., GitHub raw 

118 # CDN unreachable), don't silently overwrite existing abbreviation 

119 # data with an empty mapping. 

120 _MIN_JABREF_ABBREVS = 100 

121 if len(abbrev_to_full) < _MIN_JABREF_ABBREVS: 

122 raise RuntimeError( 

123 f"JabRef: suspiciously few abbreviations " 

124 f"({len(abbrev_to_full)} < {_MIN_JABREF_ABBREVS}); " 

125 "refusing to overwrite existing data" 

126 ) 

127 

128 output = data_dir / self.filename 

129 tmp = data_dir / f"{self.filename}.tmp" 

130 with gzip.open(tmp, "wt", encoding="utf-8") as f: 

131 json.dump({"abbrev_to_full": abbrev_to_full}, f) 

132 atomic_replace(tmp, output) 

133 

134 logger.info( 

135 f"JabRef: saved {len(abbrev_to_full)} abbreviation mappings" 

136 ) 

137 return len(abbrev_to_full)