Coverage for src/local_deep_research/web/routers/notes.py: 96%

1""" 

2Notes Routes - API endpoints and pages for the notes feature (FastAPI). 

3 

4Ported from the Flask ``web/routes/notes_routes.py`` blueprint (``notes_bp``, 

5url_prefix ``/notes``). Every endpoint preserves the same paths, methods, 

6and service calls. Three deliberate divergences from the Flask blueprint: 

7 

8 * error responses use fixed client-safe messages rather than 

9 exception-derived detail; 

10 * ``link_research_to_note`` checks that the note exists both before 

11 calling the service and again when the service raises (the Flask route 

12 recovered that 404 by matching "not found" in the exception text, which 

13 the fixed messages no longer allow; the second check covers a note 

14 deleted between the first check and the call); 

15 * the nine LLM-backed AI routes classify an unconfigured LLM separately 

16 from an invalid request and from a server fault -- 400 with 

17 ``error_type: model_not_configured`` via 

18 ``_llm_not_configured_response``, where Flask answered a generic 500 

19 (and, on ``suggest-tags``, a request-validation 400). 

20 

21Notes on the port: 

22 * ``@login_required`` → ``Depends(require_auth)``; the per-route manual 

23 ``session.get("username")`` / 401 guard is dropped because ``require_auth`` 

24 already raises 401 when there is no authenticated user. 

25 * The two blueprint-wide ``before_request`` hooks (``_reject_oversized_bodies`` 

26 and ``_reject_non_object_json_body``) are collapsed into the shared 

27 ``_notes_json_body`` dependency, injected into every mutating route. 

28 * Flask ``request.args`` → ``request.query_params``; ``session[...]`` → 

29 the ``username`` dependency / ``request.session``. 

30 * Route registration order is deliberate: the literal ``/api/notes/...`` GET 

31 routes are registered BEFORE ``GET /api/notes/{note_id}`` because Starlette 

32 matches routes in registration order (Flask's map gave static rules 

33 priority automatically; FastAPI does not). 

34""" 

35 

36import asyncio 

37import builtins 

38import itertools 

39import json 

40import math 

41import reprlib 

42 

43from fastapi import APIRouter, Depends, Request 

44from fastapi.responses import JSONResponse 

45from loguru import logger 

46from sqlalchemy.exc import IntegrityError 

47from sqlalchemy.orm import load_only 

48 

49from ...config.llm_config import ( 

50 LLM_NOT_CONFIGURED_CLIENT_MESSAGE, 

51 LLMNotConfiguredError, 

52) 

53from ...research_library.notes.services.note_ai_service import NoteAIService 

54from ...research_library.notes.services.note_service import ( 

55 MAX_TAG_LENGTH, 

56 MAX_TAGS_PER_NOTE, 

57 NOTE_CONTENT_MAX_BYTES, 

58 NoteService, 

59 _assert_content_size, 

60 escape_markdown_link_label, 

61) 

62from ...research_library.utils import handle_api_error 

63from ...security.data_sanitizer import REDACTION_TEXT 

64from ...security.log_sanitizer import ( 

65 _LOG_REDACTION_MAX_CHARS, 

66 _is_empty_log_value, 

67 _is_sensitive_log_key, 

68 _truncate_for_log_redaction, 

69 redact_for_log, 

70) 

71from ..dependencies.auth import require_auth 

72from ..dependencies.json_body import is_json_request 

73from ..dependencies.rate_limit import async_shared_limit, limiter, _user_key 

74from ..dependencies.threadpool import run_db_sync 

75from ..template_config import templates 

76 

77# Cap on user-supplied free-text query params that flow into `ILIKE %...%` 

78# over unbounded TEXT columns — shared with unified_search_routes via 

79# _search_constants so the two route modules can't diverge. The router lives 

80# in web/routers/, so reach the file in web/routes/. 

81from ..routes._search_constants import MAX_SEARCH_LEN 

82from typing import Annotated, Any 

83 

84# Bound at import time (not looked up on the class at call time) because 

85# tests monkeypatch the ``NoteAIService`` name in this module. 

86_MAX_GRADE_SOURCES = NoteAIService.MAX_GRADE_SOURCES 

87 

88MAX_LINK_TEXT_LEN = 500 

89# Per-claim length cap for the fact-check grade endpoint (claims are free text 

90# from the client; the grader bounds the count but not the per-string size). 

91MAX_CLAIM_LEN = 1000 

92# Upper bound on any ?limit query param across the list/search endpoints 

93# (was an unnamed 200 repeated at six call sites). 

94MAX_LIMIT = 200 

95 

96 

97def _clamp_limit(request: Request, default, max_=MAX_LIMIT): 

98 """Parse ``?limit`` into ``[1, max_]``, defaulting to ``default``. 

99 

100 Raises ``ValueError`` on a non-integer value; the caller wraps this in 

101 ``try`` and returns the 400 (matching the six former inline copies). 

102 """ 

103 return max(1, min(int(request.query_params.get("limit", default)), max_)) 

104 

105 

106def _clamp_text_query(value, name, max_len=MAX_SEARCH_LEN): 

107 """Return value or raise ValueError if it exceeds the cap. 

108 

109 ``None``/empty pass through unchanged so optional params behave the 

110 same as before. The caller maps the failure to a fixed 400 response. 

111 

112 Two distinct failures share the ``ValueError``: a non-string value and 

113 an over-length one. A caller that passes ``request.query_params.get(...)`` 

114 can only ever hit the length branch (Starlette's query params are always 

115 ``str``), so a single fixed length message is truthful there. A caller 

116 reading a JSON body must decide by type BEFORE calling this — see 

117 ``resolve_link`` — because the body is an uncoerced ``dict``. 

118 

119 That makes the non-string branch DEAD at every current call site (three 

120 query-param readers plus ``resolve_link``, which type-checks first). It 

121 is kept as a defensive guard, not as live behaviour: without it a future 

122 caller that forgets the type check would reach ``len(value)`` and raise 

123 ``TypeError``, which no caller here catches, turning a bad body into a 

124 500 instead of a 400. Any new call site that can pass a non-string must 

125 still do its own type check so its fixed message stays truthful. 

126 """ 

127 if value is None: 

128 return value 

129 if not isinstance(value, str): 

130 raise ValueError(f"{name} must be a string") 

131 if len(value) > max_len: 

132 raise ValueError(f"{name} exceeds maximum length ({max_len} chars)") 

133 return value 

134 

135 

136# Bounded repr for an untrusted value going into an audit-log line. A naive 

137# ``repr(value)[:100]`` slices AFTER building the full repr — for a value 

138# that came out of ``_notes_json_body`` (up to the 100 MB notes JSON body 

139# cap, see ``_MAX_JSON_BODY_BYTES`` above) that means materializing a second 

140# ~100 MB+ string (more if it's escape-heavy, since ``repr`` expands each 

141# control char to ``\xNN``) just to throw all but 100 chars away. Same 

142# problem for a huge list/dict: ``repr()`` walks and stringifies every 

143# element before any truncation happens. 

144# 

145# ``reprlib.Repr`` avoids this for STRINGS and LISTS/TUPLES only: its 

146# ``repr_str`` slices, and ``repr_list``/``repr_tuple`` ``itertools.islice``, 

147# BEFORE recursing/stringifying (see ``reprlib.Repr.repr_str`` / 

148# ``repr_list`` in the stdlib), so those costs are bounded by the limits 

149# below regardless of the input's real size. It does NOT hold for 

150# dict/set/frozenset: ``repr_dict`` / ``repr_set`` / ``repr_frozenset`` 

151# call ``reprlib._possibly_sorted()`` — a plain ``sorted()`` over the FULL 

152# container — and slice to ``maxdict``/``maxset``/``maxfrozenset`` only 

153# afterwards. That is why values reach ``reprlib`` here only through the 

154# bounded walk in ``_bound_dict_shaped_walk`` below. ``maxlevel`` also caps 

155# recursion depth for a deeply nested dict. 

156# 

157# Redaction runs BEFORE any cut. ``repr_str`` keeps a long string's first 

158# and last ~48 characters and drops the middle; the final ``[:100]`` 

159# backstop below keeps only a prefix. Either cut can drop a credential's 

160# label (``?key=``, ``api_key=``, the ``user:`` ... ``@`` around URL 

161# userinfo) while keeping part of the secret, and the loguru patcher's 

162# redaction — the backstop for every sink — then has nothing to match. The 

163# value was often logged in full before it went through this preview, so 

164# the patcher used to see the whole thing. ``_RedactingRepr`` therefore runs 

165# the patcher's own redaction (``redact_for_log``) on every string, and on 

166# every ``builtins.repr`` text ``repr_instance`` falls back to, before 

167# reprlib shortens it; ``_bound_dict_shaped_walk`` masks a value under a 

168# sensitive key name (``api_key``, ``password``, ...), as the patcher's 

169# ``extra`` redaction does, because per-string redaction cannot see a label 

170# that lives in the dict key (and masks the value under any key its 

171# key-check budget cannot cover, see ``_PREVIEW_KEY_CHECK_BUDGET``). 

172# 

173# Redaction costs about 2 microseconds per character on adversarial input 

174# (a run of spaces, or of ``https://a:b@ ``): ~70-120 ms for a 32 KiB 

175# string, where the unredacted preview cost well under 0.1 ms. The preview 

176# shows at most 100 characters, so each string is redacted only up to 

177# ``_PREVIEW_STRING_REDACTION_MAX_CHARS``: it is cut first by the 

178# patcher's own rule (``_truncate_for_log_redaction`` with that bound, 

179# only at a whitespace boundary, dropping the whole run the bound falls 

180# in), the kept head is redacted, and the rest is replaced by the 

181# omitted-length marker. A credential token holds no boundary character, 

182# so the head never keeps part of a token whose label or ``@`` lies past 

183# the cut, and nothing past the cut is shown. A bounded container can 

184# still hold over twenty thousand strings (reprlib renders up to ten keys 

185# and ten values per dict at each of four levels), so each preview also 

186# has a total redaction budget of ``_PREVIEW_REDACTION_BUDGET`` 

187# characters. Each redaction also has a fixed cost of about 25 

188# microseconds however short the string, so every call is charged at 

189# least ``_PREVIEW_REDACTION_MIN_CHARGE`` characters: a preview runs at 

190# most ``_PREVIEW_REDACTION_BUDGET // _PREVIEW_REDACTION_MIN_CHARGE`` 

191# (32) redactions over at most ``_PREVIEW_REDACTION_BUDGET`` characters 

192# in total, a few milliseconds at worst. The 100-character preview shows 

193# only the first few strings reprlib renders, and those are the ones 

194# redacted first. A string that does not fit the remaining budget is 

195# shown as ``<N-char string not shown>``, never raw. 

196_PREVIEW_STRING_REDACTION_MAX_CHARS = 512 

197_PREVIEW_REDACTION_BUDGET = 1_024 

198_PREVIEW_REDACTION_MIN_CHARGE = 32 

199 

200 

201class _RedactingRepr(reprlib.Repr): 

202 """``reprlib.Repr`` that redacts each string before shortening it. 

203 

204 Stateful (the remaining budget), so ``_log_value_preview`` builds a new 

205 one per call; the module-level ``_log_value_repr`` instance only holds 

206 the shared limits. 

207 """ 

208 

209 def __init__(self) -> None: 

210 super().__init__( 

211 maxlevel=4, 

212 maxtuple=10, 

213 maxlist=10, 

214 maxdict=10, 

215 maxset=10, 

216 maxfrozenset=10, 

217 maxstring=100, 

218 maxlong=100, 

219 maxother=100, 

220 ) 

221 self._budget = _PREVIEW_REDACTION_BUDGET 

222 

223 def _redacted(self, text: str): 

224 cost = max( 

225 min(len(text), _PREVIEW_STRING_REDACTION_MAX_CHARS), 

226 _PREVIEW_REDACTION_MIN_CHARGE, 

227 ) 

228 if cost > self._budget: 

229 return None 

230 self._budget -= cost 

231 head, marker = _truncate_for_log_redaction( 

232 text, _PREVIEW_STRING_REDACTION_MAX_CHARS 

233 ) 

234 return redact_for_log(head) + marker 

235 

236 def repr_str(self, x, level): 

237 redacted = self._redacted(x) 

238 if redacted is None: 

239 return f"<{len(x)}-char string not shown>" 

240 return super().repr_str(redacted, level) 

241 

242 def repr_instance(self, x, level): 

243 try: 

244 text = builtins.repr(x) 

245 except Exception: 

246 return f"<{x.__class__.__name__} instance at {id(x):#x}>" 

247 redacted = self._redacted(text) 

248 if redacted is None: 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true

249 return f"<{x.__class__.__name__} repr not shown>" 

250 if len(redacted) > self.maxother: 250 ↛ 251line 250 didn't jump to line 251 because the condition on line 250 was never true

251 i = max(0, (self.maxother - 3) // 2) 

252 j = max(0, self.maxother - 3 - i) 

253 redacted = ( 

254 redacted[:i] + self.fillvalue + redacted[len(redacted) - j :] 

255 ) 

256 return redacted 

257 

258 

259_log_value_repr = _RedactingRepr() 

260 

261 

262def _log_value_preview(value: Any) -> str: 

263 """Bounded, ``logger.warning``-safe preview of an untrusted JSON value. 

264 

265 Use in place of ``repr(value)[:100]`` at any audit-log site whose value 

266 comes from request input (query params, and especially JSON bodies, 

267 which this router allows up to ~100 MB). See ``_log_value_repr`` above 

268 for why plain ``repr()[:100]`` is unsafe here. 

269 

270 ``value`` is pre-bounded by ``_bound_dict_shaped_walk`` before it 

271 reaches ``reprlib`` — see that function for why reprlib's own limits do 

272 not bound dict/set/frozenset cost on their own. 

273 

274 The final bound is a cheap backstop, not the primary defense — the 

275 limits on ``_log_value_repr`` plus the pre-bounding step already keep 

276 the *work* bounded; this just keeps the *output* bounded too (a 

277 container's ten per-element reprs, plus reprlib's own ``...`` marker, 

278 can still sum past 100 chars). But the backstop must not slice 

279 truncation *markers* off with the content, and it must keep the two 

280 truncation conditions distinguishable. When it truncates, the preview 

281 gets one of exactly two endings, each carrying its own meaning: 

282 

283 * an items-omitted tail — ``', ...}'``, ``', ...]'`` or ``', ...)'``: 

284 ENTRIES were dropped from some container (it held strictly more 

285 than ``reprlib``'s per-container limit of 10). A repr that already 

286 ends in one of ``reprlib``'s own container tails keeps it inside 

287 the cap; a repr whose dropped entries are NESTED — the tail sits 

288 inside the repr, followed by the outer container's closing 

289 delimiters, so it is not the suffix of the whole repr and the 

290 tail check cannot see it — instead gets 

291 ``_PREVIEW_ITEMS_OMITTED_TAIL`` appended, guided by the 

292 ``items_omitted`` flag the pre-bounding walk reports. 

293 * a bare ``...`` (``_PREVIEW_ELLIPSIS``): LENGTH-only truncation — 

294 no visited container lost entries; the repr simply exceeded the 

295 char cap. 

296 

297 The two endings differ in their final character and so can never be 

298 byte-equal: an 11-key (or 50,010-key) payload nested anywhere the 

299 pre-bounding walk visits always previews differently from the 

300 corresponding all-≤10-key payload. Reprs that fit the cap are 

301 returned untouched, and the preview never exceeds the cap. 

302 

303 Every string (and every ``repr`` text of a non-container object) has 

304 already been through the log patcher's credential redaction before 

305 ``reprlib`` or the backstop shortens it, and a value under a sensitive 

306 dict key (or under a key the key-check budget could not cover) is 

307 masked whole — see ``_RedactingRepr`` above. A cut 

308 therefore only removes text the patcher already redacted or would 

309 leave as is: it cannot strip a recognised credential's label while 

310 keeping part of the secret. 

311 """ 

312 bounded, items_omitted = _bound_dict_shaped_walk(value, 0) 

313 text = _RedactingRepr().repr(bounded) 

314 if len(text) <= _PREVIEW_MAX_CHARS: 

315 return text 

316 for tail in _PREVIEW_CONTAINER_TAILS: 

317 if text.endswith(tail): 

318 return text[: _PREVIEW_MAX_CHARS - len(tail)] + tail 

319 if items_omitted: 

320 return ( 

321 text[: _PREVIEW_MAX_CHARS - len(_PREVIEW_ITEMS_OMITTED_TAIL)] 

322 + _PREVIEW_ITEMS_OMITTED_TAIL 

323 ) 

324 return ( 

325 text[: _PREVIEW_MAX_CHARS - len(_PREVIEW_ELLIPSIS)] + _PREVIEW_ELLIPSIS 

326 ) 

327 

328 

329# One MORE than ``_log_value_repr``'s own maxdict/maxlist/maxset/maxtuple 

330# (10), deliberately. reprlib only appends its ``...`` truncation marker 

331# when the container handed to it is STRICTLY longer than that limit (``if 

332# n > self.maxdict: pieces.append(self.fillvalue)``). Pre-truncating to 

333# exactly 10 would make an attacker's 400,000-key body render byte-identical 

334# to a legitimate 10-key one — silently erasing "there was more" from the 

335# audit line, which is the one thing this log site exists to record. Keeping 

336# 11 costs nothing (reprlib discards the 11th) and restores the marker. 

337# ``_PREVIEW_MAX_DEPTH`` matches ``_log_value_repr``'s ``maxlevel``. 

338_PREVIEW_MAX_ITEMS = 11 

339_PREVIEW_MAX_DEPTH = 4 

340# Final output cap for ``_log_value_preview`` (previously an inline 

341# ``[:100]`` slice). 

342_PREVIEW_MAX_CHARS = 100 

343# Matches reprlib's ``fillvalue`` and the ``...`` marker convention used 

344# throughout this module. 

345_PREVIEW_ELLIPSIS = "..." 

346# reprlib's "the container had more items than I showed" tails — its ``...`` 

347# fillvalue joined onto the container's closing bracket. When the final 

348# char bound bites, preserving the tail keeps an item-bounded payload (e.g. 

349# a 50,010-key dict whose first ten ~100-char values fill the cap on their 

350# own) distinguishable from a merely char-bounded one, instead of slicing 

351# the marker off entirely. 

352_PREVIEW_CONTAINER_TAILS = (", ...}", ", ...]", ", ...)") 

353# The nested counterpart to the tails above. When the pre-bounding walk 

354# reports that some container it visited held more entries than 

355# ``reprlib``'s per-container limit (so entries were dropped) but the 

356# ``reprlib`` fill sits INSIDE the repr — followed by the outer 

357# container's closing delimiters, not at the very end — the tail check 

358# in ``_log_value_preview`` cannot preserve it. Appending this explicit 

359# tail keeps such a payload (e.g. ``{"outer": {<11 keys with ~100-char 

360# values>}}``) byte-distinguishable from the corresponding all-≤10-key 

361# payload, whose only truncation is the char cap and which therefore 

362# ends in a bare ``...``. Deliberately identical to the dict tail: both 

363# spell "entries were omitted". 

364_PREVIEW_ITEMS_OMITTED_TAIL = ", ...}" 

365# Budget for the sensitive-key check in ``_bound_dict_shaped_walk``. The 

366# check walks the whole key (``DataSanitizer.is_sensitive_setting`` 

367# normalises it character by character), JSON keys have no length limit, 

368# and the walk can visit over sixteen thousand keys (up to 11 per dict in 

369# up to 1,464 dicts). Each check of a str key is charged the key's length, 

370# at least ``_PREVIEW_KEY_CHECK_MIN_CHARGE`` (a non-str key is never 

371# sensitive, and its check is free), so one preview checks at most 

372# ``_PREVIEW_KEY_CHECK_BUDGET`` characters of keys in at most 512 checks. 

373# A key the remaining budget cannot cover is not checked, and its value is 

374# masked as if the key were sensitive (fail closed). 

375_PREVIEW_KEY_CHECK_BUDGET = _LOG_REDACTION_MAX_CHARS 

376_PREVIEW_KEY_CHECK_MIN_CHARGE = 64 

377 

378 

379def _bound_dict_shaped_walk( 

380 value: Any, _depth: int, _key_budget: list[int] | None = None 

381) -> tuple[Any, bool]: 

382 """Return a size-bounded copy of ``value``, safe to hand to 

383 ``_log_value_repr.repr()``, plus an "entries were dropped" flag. 

384 

385 ``reprlib.Repr`` bounds *string* and *list/tuple* reprs by slicing 

386 BEFORE recursing (see ``repr_str`` / ``repr_list`` in the stdlib), so 

387 those are already O(1) regardless of input size — the module comment 

388 above relies on that. But ``repr_dict`` / ``repr_set`` / 

389 ``repr_frozenset`` call stdlib ``reprlib._possibly_sorted()`` — which is 

390 a plain ``sorted(x)`` over the FULL container — and only slice to 

391 ``maxdict``/``maxset``/``maxfrozenset`` afterwards. For a dict/set built 

392 from a large attacker-controlled JSON body (this router allows up to 

393 ~100 MB, e.g. ``{"is_collapsed": {<N keys>}}`` reaching 

394 ``patch_note_research``) that reintroduces the exact O(N log N) time / 

395 O(N) memory cost ``_log_value_repr`` exists to eliminate — measured at 

396 ~10-22 ms for a 400k-entry dict/set on py3.14, scaling linearly, versus 

397 ~0 ms for an equivalently large string or list through the same helper. 

398 

399 Fix: truncate every dict/set/frozenset to ``_PREVIEW_MAX_ITEMS`` entries 

400 OURSELVES — via ``itertools.islice`` over the container's natural 

401 iteration order (insertion order for dicts, arbitrary-but-cheap for 

402 sets), never ``sorted()`` — before reprlib ever sees it. Once bounded 

403 this way, reprlib's own ``repr_dict``/``repr_set`` still calls 

404 ``sorted()``, but over at most ``_PREVIEW_MAX_ITEMS`` items, which is 

405 negligible. 

406 

407 Recurses into dict values and list/tuple elements (only the first 

408 ``_PREVIEW_MAX_ITEMS`` of each, mirroring reprlib's own slicing) up to 

409 ``_PREVIEW_MAX_DEPTH`` — matching ``_log_value_repr``'s ``maxlevel`` — 

410 so a large dict/set nested a few levels down (e.g. inside a list inside 

411 a dict) is bounded too, not just one sitting at the top level. The 

412 number of containers copied is the geometric sum 

413 ``1 + M + M**2 + M**3`` for ``M = _PREVIEW_MAX_ITEMS`` (1,464 at the 

414 current limits), NOT ``M ** _PREVIEW_MAX_DEPTH``, regardless of the real 

415 structure's size. Dict/set *keys* are returned unbounded: JSON object 

416 keys are always strings, and rendering a huge key is bounded by 

417 ``_RedactingRepr``'s redaction budget and ``repr_str`` slicing. The 

418 sensitive-key check on each visited dict key is NOT free — it walks the 

419 whole key — so it draws on a per-preview budget, 

420 ``_PREVIEW_KEY_CHECK_BUDGET`` (``_key_budget`` holds what remains; the 

421 outermost call creates it). A key the remaining budget cannot cover is 

422 not checked and its value is masked whole, as for a sensitive key. The 

423 walk still descends into that masked value (with no budget, so every 

424 key under it is masked unchecked) to compute the items-omitted flag. 

425 

426 Past ``_PREVIEW_MAX_DEPTH`` the walk stops and the sub-value is handed 

427 to reprlib untouched. For a **dict** that costs nothing: ``repr_dict`` 

428 checks ``level <= 0`` *before* calling ``_possibly_sorted``, so an 

429 over-deep dict collapses to ``{...}`` unsorted. It is NOT free for a 

430 **set/frozenset**: ``repr_set``/``repr_frozenset`` call 

431 ``_possibly_sorted`` at the top of the method, *before* 

432 ``_repr_iterable`` gets to check ``level``, so a huge set nested past 

433 the cap is still sorted in full and the result thrown away (measured 

434 ~13 ms for ``[[[[set(range(400_000))]]]]``, output ``[[[[{...}]]]]``). 

435 Likewise, set/frozenset *elements* are never recursed into, so a 

436 container inside a set — a frozenset or tuple, the only hashable 

437 candidates — stays unbounded at any depth (~13 ms for 

438 ``{frozenset(range(400_000))}``). Neither shape is reachable from this 

439 router (JSON has no set literal), but both matter if this helper is 

440 reused elsewhere. 

441 

442 Values that are not dict/set/frozenset/list/tuple (str, int, float, 

443 bool, None) pass through unchanged — reprlib's ``maxstring`` / 

444 ``maxlong`` bound those. ``maxlong`` only slices an ALREADY-built 

445 string, though: ``reprlib.repr_int`` calls ``builtins.repr(x)`` in full 

446 before truncating, so a huge ``int`` is the one JSON-native type whose 

447 repr cost this helper does not bound (measured ~2 s for a 2,000,000- 

448 digit int). It is unreachable from this router only because 

449 ``json.loads`` refuses an integer literal longer than 

450 ``sys.get_int_max_str_digits()`` (4300 by default). Nor does the 

451 pass-through extend to an arbitrary object: reprlib has no per-type 

452 method for a custom class, so it falls through to ``repr_instance``, 

453 which calls ``builtins.repr(x)`` in FULL and ignores ``level`` 

454 entirely — a hostile ``__repr__`` can allocate whatever it likes 

455 (measured ~170 ms / 200 MB) before reprlib truncates the resulting 

456 string. Nothing here bounds that. It is unreachable at this router's 

457 call sites, whose values all come out of ``json.loads`` and so can only 

458 be JSON-native types, but any reuse of this helper at a call site 

459 logging a non-JSON-sourced value must account for it. 

460 

461 Returns ``(bounded_value, items_omitted)``. ``items_omitted`` is True 

462 iff any dict/set/frozenset/list/tuple visited at a depth ``reprlib`` 

463 still enumerates (below ``_PREVIEW_MAX_DEPTH``, which matches its 

464 ``maxlevel``) holds at least ``_PREVIEW_MAX_ITEMS`` entries — i.e. 

465 strictly more than ``reprlib``'s per-container limit of one less, the 

466 condition under which ``reprlib`` drops entries and appends its 

467 ``...`` fill. ``_log_value_preview`` uses the flag to give over-cap 

468 reprs whose item truncation is nested (fill not at the end of the 

469 repr) the explicit ``_PREVIEW_ITEMS_OMITTED_TAIL`` ending. 

470 Containers at or past the depth cap are handed to ``reprlib`` 

471 untouched and collapse there to ``{...}`` / ``[...]`` wholesale — no 

472 per-entry output at any item count — so they do not set the flag. 

473 """ 

474 if _depth >= _PREVIEW_MAX_DEPTH: 

475 return value, False 

476 if _key_budget is None: 

477 _key_budget = [_PREVIEW_KEY_CHECK_BUDGET] 

478 if isinstance(value, dict): 

479 bounded: dict[Any, Any] = {} 

480 omitted = len(value) >= _PREVIEW_MAX_ITEMS 

481 for k, v in itertools.islice(value.items(), _PREVIEW_MAX_ITEMS): 

482 # A non-str key is never sensitive and costs nothing to check. 

483 cost = ( 

484 max(len(k), _PREVIEW_KEY_CHECK_MIN_CHARGE) 

485 if isinstance(k, str) 

486 else 0 

487 ) 

488 if cost > _key_budget[0]: 

489 # Not checked (see ``_PREVIEW_KEY_CHECK_BUDGET``), so 

490 # masked unless empty. The walk below only computes the 

491 # flag: with no budget, everything under it is masked too. 

492 if not _is_empty_log_value(v): 492 ↛ 507line 492 didn't jump to line 507 because the condition on line 492 was always true

493 bounded[k] = REDACTION_TEXT 

494 _, child_omitted = _bound_dict_shaped_walk( 

495 v, _depth + 1, [0] 

496 ) 

497 omitted = omitted or child_omitted 

498 continue 

499 else: 

500 _key_budget[0] -= cost 

501 if _is_sensitive_log_key(k) and not _is_empty_log_value(v): 

502 # Masked whole, as the log patcher masks a sensitive 

503 # ``extra`` key: string redaction cannot see a label 

504 # that lives in the key (see ``_RedactingRepr``). 

505 bounded[k] = REDACTION_TEXT 

506 continue 

507 bounded[k], child_omitted = _bound_dict_shaped_walk( 

508 v, _depth + 1, _key_budget 

509 ) 

510 omitted = omitted or child_omitted 

511 return bounded, omitted 

512 if isinstance(value, (set, frozenset)): 

513 # Bound the count only — set/frozenset elements are NOT recursed 

514 # into (see the docstring: a frozenset or tuple nested inside a set 

515 # therefore stays unbounded; unreachable from JSON input, which has 

516 # no set literal). Rebuilt as a plain ``set``/``frozenset`` rather 

517 # than ``type(value)(...)``, because a set SUBCLASS with a custom 

518 # ``__init__`` would come back silently EMPTY or raise ``TypeError`` 

519 # from inside a logging call — turning a 400 into a 500. 

520 bounded_items = itertools.islice(value, _PREVIEW_MAX_ITEMS) 

521 if isinstance(value, frozenset): 

522 return frozenset(bounded_items), len(value) >= _PREVIEW_MAX_ITEMS 

523 return set(bounded_items), len(value) >= _PREVIEW_MAX_ITEMS 

524 if isinstance(value, (list, tuple)): 

525 bounded = [] 

526 omitted = len(value) >= _PREVIEW_MAX_ITEMS 

527 for v in itertools.islice(value, _PREVIEW_MAX_ITEMS): 

528 bounded_item, child_omitted = _bound_dict_shaped_walk( 

529 v, _depth + 1, _key_budget 

530 ) 

531 bounded.append(bounded_item) 

532 omitted = omitted or child_omitted 

533 # Rebuilt as a plain ``tuple``, so a namedtuple degrades to a 

534 # bare tuple in the preview — acceptable for an audit line. 

535 bounded_tuple = tuple(bounded) if isinstance(value, tuple) else bounded 

536 return bounded_tuple, omitted 

537 return value, False 

538 

539 

540def _llm_not_configured_response(operation: str) -> JSONResponse: 

541 """400 for a notes AI route whose LLM is not configured. 

542 

543 Nine routes in this file run an LLM (``summarize``, 

544 ``research-questions``, ``suggest-tags``, ``key-concepts``, 

545 ``fact-check``, ``fact-check/.../grade``, ``synthesize/preview``, 

546 ``synthesize``, ``versions/semantic-diff``). Every one of them reaches 

547 ``NoteAIService._get_llm`` -> ``config.llm_config.get_llm``, which raises 

548 ``LLMNotConfiguredError`` when the user has not chosen a provider/model 

549 yet -- and on a fresh install that is the FIRST thing these endpoints 

550 hit. Left to ``handle_api_error`` they answer 500 "contact support"; in 

551 ``suggest_tags``, whose handler also maps ``ValueError`` to a 

552 request-validation 400, it came back as a bad-request verdict on a body 

553 that was perfectly fine. Both send the user looking in the wrong place. 

554 

555 400 with ``error_type: model_not_configured`` is this repository's 

556 existing answer for an unconfigured LLM, not a 5xx: see 

557 ``web/routers/api.py::check_ollama_model`` (the same ``error_type`` for 

558 an empty ``llm.model``), ``web/routers/followup.py::start_followup`` 

559 (400 before any DB write when ``llm.model`` is empty), and 

560 ``web/routers/zotero.py::_not_configured_response`` -- "that is a 

561 client-state problem, not a server error". The 503s in ``rag.py`` and 

562 ``settings.py`` are NOT the precedent: those are egress-policy 

563 ``settings_unavailable`` arms, a different condition. 

564 

565 One helper rather than nine copies so the status, the message and the 

566 ``error_type`` cannot drift apart route by route. ``operation`` names 

567 the route in the server-side log only; it never reaches the client, and 

568 the client message is always the module constant -- never ``str(exc)``, 

569 whose setup guidance can carry a settings path. 

570 """ 

571 logger.warning("Notes AI: {} refused, LLM not configured", operation) 

572 return JSONResponse( 

573 { 

574 "success": False, 

575 "error": LLM_NOT_CONFIGURED_CLIENT_MESSAGE, 

576 "error_type": "model_not_configured", 

577 }, 

578 status_code=400, 

579 ) 

580 

581 

582# Rate limits for AI-heavy endpoints. Split into two buckets so cheap 

583# FAISS / local-embedding lookups (semantic search, similar notes, 

584# suggested links, related research) don't drain the same budget as 

585# expensive LLM calls (summarize, suggest_tags, extract_key_concepts, 

586# semantic_diff). A user typing through the autocomplete UI shouldn't 

587# block a single LLM call for a full minute. 

588# 

589# ``key_func=_user_key`` keys each bucket per authenticated user; without it 

590# the global Limiter default (client IP) would let shared-IP users (NAT, lab, 

591# reverse-proxy-without-trusted-forward-headers) drain each other's 

592# LLM/synthesis budgets. 

593_notes_ai_limit = async_shared_limit( 

594 limiter, "10 per minute", scope="notes_ai", key_func=_user_key 

595) 

596_notes_search_limit = limiter.shared_limit( 

597 "60 per minute", scope="notes_search", key_func=_user_key 

598) 

599_notes_synthesize_limit = async_shared_limit( 

600 limiter, "5 per minute", scope="notes_synthesize", key_func=_user_key 

601) 

602# Notes write budget — applies to create_note, update_note, delete_note, 

603# and add/remove from collection. 60/min comfortably covers active 

604# editing (a save every second for a minute) but blocks scripted abuse: 

605# pre-fix an authenticated user could rapid-POST PUT /api/notes/<id> 

606# under the global IP cap (5000/hr) and saturate the 4-worker 

607# summary_executor with bursty LLM change-summary tasks, queueing 

608# hundreds of MB of (old, new) content pairs. Per-user keying so two 

609# real users on shared NAT don't compete for the same bucket. 

610_notes_write_limit = limiter.shared_limit( 

611 "60 per minute", scope="notes_write", key_func=_user_key 

612) 

613# Save-as-note copies a whole completed report (up to the 50MB content 

614# cap) into a fresh note + version snapshot per call. At the generic 

615# notes_write 60/min that's on the order of GBs/min of per-user DB writes 

616# from one client — a disk/backup-bloat vector. Tighter, cost-aware bucket. 

617_notes_save_as_note_limit = limiter.shared_limit( 

618 "10 per minute", scope="notes_save_as_note", key_func=_user_key 

619) 

620# Fact-check kicks off a full LDR research run downstream, so it's strictly 

621# tighter than the LLM bucket — a few per minute bounds research load. 

622_notes_factcheck_limit = async_shared_limit( 

623 limiter, "3 per minute", scope="notes_factcheck", key_func=_user_key 

624) 

625 

626router = APIRouter(prefix="/notes", tags=["notes"]) 

627 

628# Pre-parse request-body ceiling for this JSON API surface. The app-wide 

629# request size limit is sized for multi-file uploads (hundreds of GB), and 

630# parsing buffers the WHOLE body before any service validation runs — so 

631# without this check a multi-GB JSON body reached memory before 

632# _assert_content_size could reject it at 50 MB. 

633# 

634# Budget 2× the content cap. This bounds pre-parse memory; it does NOT 

635# precisely enforce the 50 MB content limit (that stays in 

636# _assert_content_size after parsing). 2× covers the realistic worst case 

637# where every character needs a 2-byte JSON escape (``"`` / ``\`` / 

638# newline); content dominated by control chars (each expands to a 6-byte 

639# ``\uXXXX``) can still exceed this and get a 413 even though its decoded 

640# size is under 50 MB — an acceptable trade for keeping the cap far below 

641# a memory-exhausting multi-GB body. 

642_MAX_JSON_BODY_BYTES = 2 * NOTE_CONTENT_MAX_BYTES 

643 

644 

645#: Above this size the JSON parse is handed to a worker thread. Measured on 

646#: the pinned interpreter with an adversarial-but-valid document (many small 

647#: objects, which is what an attacker sends -- not one long string, which is 

648#: what a benign client sends), ``json.loads`` runs at ~110 ms/MB and scales 

649#: linearly. So 64 KiB is ~7 ms on the loop, and the notes ceiling of 100 MB 

650#: would be ~11 SECONDS -- a whole-instance freeze under single-worker 

651#: uvicorn, where Flask's thread-per-request model only cost one thread. 

652#: Below the threshold the parse is sub-10 ms and dispatching it would trade 

653#: that for contention on the same threadpool the DB work uses. 

654_JSON_OFFLOAD_THRESHOLD_BYTES = 64 * 1024 

655 

656 

657async def _parse_json_off_loop(body): 

658 """``json.loads`` without stalling the event loop on a large body. 

659 

660 ``asyncio.to_thread`` rather than ``run_db_sync``: this opens no DB 

661 session, so the session cleanup that wrapper does is unnecessary 

662 (see dependencies/threadpool.py::run_db_sync). 

663 

664 The bytearray is passed through without copying to ``bytes`` -- at the 

665 notes cap that copy would itself be a 100 MB allocation, and nothing 

666 mutates the buffer after this point. 

667 """ 

668 if len(body) < _JSON_OFFLOAD_THRESHOLD_BYTES: 

669 return json.loads(body) 

670 return await asyncio.to_thread(json.loads, body) 

671 

672 

673async def _notes_json_body( 

674 request: Request, 

675) -> dict[str, Any] | JSONResponse: 

676 """Shared body gate for every mutating notes route. 

677 

678 Replaces the blueprint's two ``before_request`` hooks: 

679 

680 * 413 a body whose ``Content-Length`` exceeds ``_MAX_JSON_BODY_BYTES`` 

681 (bounds pre-parse memory before ``request.json()`` buffers it). 

682 * 400 a declared-JSON body that parses to something other than an 

683 object — ``true`` / ``123`` / ``"x"`` / ``[1, 2]`` would otherwise 

684 slip past the routes' ``... or {}`` idioms and crash on the first 

685 ``.get`` as an opaque 500. 

686 

687 Returns either a ``JSONResponse`` (the caller must return it verbatim) or 

688 the parsed dict. A missing/empty/malformed body becomes ``{}`` so each 

689 route applies its own "no data provided" default exactly as before. 

690 """ 

691 

692 def _too_large(): 

693 return JSONResponse( 

694 { 

695 "success": False, 

696 "error": ( 

697 "Request body too large " 

698 f"(max {_MAX_JSON_BODY_BYTES // (1024 * 1024)} MB)" 

699 ), 

700 }, 

701 status_code=413, 

702 ) 

703 

704 # Fast path: reject a body that DECLARES an over-cap Content-Length before 

705 # reading anything. 

706 content_length = request.headers.get("content-length") 

707 if content_length is not None: 

708 try: 

709 declared = int(content_length) 

710 except (TypeError, ValueError): 

711 declared = None 

712 if declared is not None and declared > _MAX_JSON_BODY_BYTES: 

713 return _too_large() 

714 

715 # Read the body from the stream with a running cap, so a chunked request 

716 # (``Transfer-Encoding: chunked``, no Content-Length) is cut off at the 

717 # notes ceiling BEFORE it is fully buffered — the Flask blueprint got this 

718 # from ``arm_notes_body_cap`` setting ``request.max_content_length`` at the 

719 # stream level; the plain Content-Length check above cannot (a chunked 

720 # body has no Content-Length to inspect). No notes route reads the body 

721 # again after this dependency, so consuming the stream here is safe. 

722 body = bytearray() 

723 async for chunk in request.stream(): 

724 body.extend(chunk) 

725 if len(body) > _MAX_JSON_BODY_BYTES: 

726 return _too_large() 

727 

728 # Keep the body cap for every media type. Flask's get_json(silent=True) 

729 # then treated a non-JSON media type as no data, even when its bytes 

730 # looked like JSON. 

731 if not is_json_request(request): 

732 return {} 

733 

734 if not body: 734 ↛ 737line 734 didn't jump to line 737 because the condition on line 734 was never true

735 # Missing/empty body — leave the route to default as it sees fit 

736 # (matches Flask's ``get_json(silent=True) -> None``). 

737 return {} 

738 try: 

739 data = await _parse_json_off_loop(body) 

740 except Exception: 

741 # Malformed JSON — same "leave the route to default" behavior. 

742 return {} 

743 

744 if not isinstance(data, dict): 

745 return JSONResponse( 

746 { 

747 "success": False, 

748 "error": "Request body must be a JSON object", 

749 }, 

750 status_code=400, 

751 ) 

752 return data 

753 

754 

755_NotesBody = Annotated[ 

756 dict[str, Any] | JSONResponse, 

757 Depends(_notes_json_body), 

758] 

759 

760 

761def _trigger_note_auto_index(note_id, service, username, session_id): 

762 """Trigger async auto-indexing for a note (non-blocking).""" 

763 try: 

764 from ...web.routers.rag import trigger_auto_index 

765 from ...database.session_passwords import session_password_store 

766 

767 db_password = session_password_store.get_session_password( 

768 username, session_id 

769 ) 

770 if not db_password: 

771 logger.debug( 

772 "Skipping note auto-index for {}: no session password available", 

773 note_id, 

774 ) 

775 return 

776 collection_id = service.get_notes_collection_id() 

777 if not collection_id: 

778 logger.debug( 

779 "Skipping note auto-index for {}: no notes collection", 

780 note_id, 

781 ) 

782 return 

783 trigger_auto_index([note_id], collection_id, username, db_password) 

784 except Exception: 

785 logger.exception("Failed to trigger auto-indexing for note") 

786 

787 

788# ============================================================================ 

789# Page Routes 

790# ============================================================================ 

791 

792 

793@router.get("/") 

794def notes_page( 

795 request: Request, username: Annotated[str, Depends(require_auth)] 

796): 

797 """Render the notes list page.""" 

798 return templates.TemplateResponse( 

799 request=request, 

800 name="pages/notes.html", 

801 context={"active_page": "notes"}, 

802 ) 

803 

804 

805@router.get("/{note_id}") 

806def note_detail_page( 

807 request: Request, 

808 note_id: str, 

809 username: Annotated[str, Depends(require_auth)], 

810): 

811 """Render the note detail page.""" 

812 from ..utils.path_ids import validated_uuid_path_param 

813 

814 note_id = validated_uuid_path_param(note_id, "Note") 

815 return templates.TemplateResponse( 

816 request=request, 

817 name="pages/note_detail.html", 

818 context={"active_page": "notes", "note_id": note_id}, 

819 ) 

820 

821 

822# ============================================================================ 

823# API Routes - Notes CRUD 

824# ============================================================================ 

825 

826 

827@router.get("/api/notes") 

828@_notes_search_limit 

829def list_notes( 

830 request: Request, username: Annotated[str, Depends(require_auth)] 

831): 

832 """ 

833 List notes with optional filtering. 

834 

835 Query params: 

836 collection_id: Filter by collection 

837 search: Search in title and content (capped at MAX_SEARCH_LEN) 

838 pinned_only: Only return pinned notes 

839 limit: Maximum number (default 100) 

840 offset: Offset for pagination (default 0) 

841 """ 

842 try: 

843 service = NoteService(username) 

844 

845 collection_id = request.query_params.get("collection_id") 

846 try: 

847 search = _clamp_text_query( 

848 request.query_params.get("search"), "search" 

849 ) 

850 except ValueError: 

851 return JSONResponse( 

852 { 

853 "success": False, 

854 "error": ( 

855 f"search exceeds maximum length " 

856 f"({MAX_SEARCH_LEN} chars)" 

857 ), 

858 }, 

859 status_code=400, 

860 ) 

861 pinned_only = ( 

862 request.query_params.get("pinned_only", "").lower() == "true" 

863 ) 

864 try: 

865 limit = _clamp_limit(request, 100) 

866 offset = max(0, int(request.query_params.get("offset", 0))) 

867 except (ValueError, TypeError): 

868 return JSONResponse( 

869 {"success": False, "error": "Invalid limit or offset"}, 

870 status_code=400, 

871 ) 

872 

873 # Total under the same filter — lets the frontend render a real 

874 # page-N-of-M control rather than guessing from len(notes) == 

875 # limit. count_notes repeats the filtered scan list_notes runs 

876 # (for a keyword search that is a second full ILIKE pass over 

877 # text_content), so skip it when a short first page already 

878 # reveals the total. 

879 if offset == 0: 

880 notes = service.list_notes( 

881 collection_id=collection_id, 

882 search=search, 

883 pinned_only=pinned_only, 

884 limit=limit, 

885 offset=0, 

886 ) 

887 total = ( 

888 len(notes) 

889 if len(notes) < limit 

890 else service.count_notes( 

891 collection_id=collection_id, 

892 search=search, 

893 pinned_only=pinned_only, 

894 ) 

895 ) 

896 else: 

897 total = service.count_notes( 

898 collection_id=collection_id, 

899 search=search, 

900 pinned_only=pinned_only, 

901 ) 

902 # Clamp an offset past the end of the result set: it returns 

903 # an empty page anyway, so don't make the DB scan-and-discard 

904 # `offset` rows for a guaranteed-empty result (mirrors 

905 # get_note_versions). 

906 if offset > total: 

907 offset = total 

908 

909 notes = service.list_notes( 

910 collection_id=collection_id, 

911 search=search, 

912 pinned_only=pinned_only, 

913 limit=limit, 

914 offset=offset, 

915 ) 

916 

917 return { 

918 "success": True, 

919 "notes": notes, 

920 "total": total, 

921 "limit": limit, 

922 "offset": offset, 

923 } 

924 

925 except Exception as e: 

926 return handle_api_error("listing notes", e) 

927 

928 

929@router.post("/api/notes") 

930@_notes_write_limit 

931def create_note( 

932 request: Request, 

933 username: Annotated[str, Depends(require_auth)], 

934 body: _NotesBody, 

935): 

936 """ 

937 Create a new note. 

938 

939 Body: 

940 title: Note title (required) 

941 content: Note content (required) 

942 tags: List of tags (optional) 

943 collection_ids: List of collection IDs (optional) 

944 """ 

945 if isinstance(body, JSONResponse): 

946 return body 

947 

948 try: 

949 data = body 

950 if not isinstance(data, dict) or not data: 

951 # get_json returns the parsed body verbatim, so a top-level JSON 

952 # array/string/number would pass a bare truthiness check and then 

953 # crash data.get() with AttributeError → opaque 500. Require an 

954 # object. 

955 return JSONResponse( 

956 {"success": False, "error": "No data provided"}, 

957 status_code=400, 

958 ) 

959 

960 title = data.get("title") 

961 content = data.get("content") 

962 

963 if not title: 

964 return JSONResponse( 

965 {"success": False, "error": "Title is required"}, 

966 status_code=400, 

967 ) 

968 if not content: 

969 return JSONResponse( 

970 {"success": False, "error": "Content is required"}, 

971 status_code=400, 

972 ) 

973 

974 service = NoteService(username) 

975 note_id = service.create_note( 

976 title=title, 

977 content=content, 

978 tags=data.get("tags"), 

979 collection_ids=data.get("collection_ids"), 

980 ) 

981 

982 # Trigger async auto-indexing (non-blocking) 

983 _trigger_note_auto_index( 

984 note_id, service, username, request.session.get("session_id") 

985 ) 

986 

987 return JSONResponse({"success": True, "id": note_id}, status_code=201) 

988 

989 except ValueError: 

990 # Never serialize exception text at the HTTP boundary. Even callers 

991 # that normally raise fixed validation messages can regress to an 

992 # exception containing paths, SQL, credentials, or library internals. 

993 return JSONResponse( 

994 {"success": False, "error": "Invalid note data"}, status_code=400 

995 ) 

996 except Exception as e: 

997 return handle_api_error("creating note", e) 

998 

999 

1000# --------------------------------------------------------------------------- 

1001# Literal-path GET routes under /api/notes/... MUST be registered before 

1002# GET /api/notes/{note_id} below — Starlette matches routes in registration 

1003# order, so otherwise ``/api/notes/semantic-search`` etc. would be captured 

1004# by the ``{note_id}`` route. (Flask's URL map gave static rules priority 

1005# automatically; FastAPI does not.) 

1006# --------------------------------------------------------------------------- 

1007 

1008 

1009@router.get("/api/notes/semantic-search") 

1010@_notes_search_limit 

1011def semantic_search_notes( 

1012 request: Request, username: Annotated[str, Depends(require_auth)] 

1013): 

1014 """ 

1015 Search notes using semantic similarity. 

1016 

1017 Finds notes by meaning rather than keyword matching. 

1018 

1019 Query params: 

1020 q: Search query (required) 

1021 limit: Maximum results (default 10) 

1022 min_similarity: Minimum similarity threshold 0.0-1.0 (default 0.3) 

1023 collection_id: Optional — restrict results to this collection's notes 

1024 (matches the text-search filter). 

1025 """ 

1026 try: 

1027 try: 

1028 query = _clamp_text_query(request.query_params.get("q", ""), "q") 

1029 except ValueError: 

1030 return JSONResponse( 

1031 { 

1032 "success": False,