Coverage for src/local_deep_research/research_library/notes/services/note_service.py: 92%
941 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-06 15:42 +0000
1"""
2Note Service - CRUD operations for notes stored as Documents.
4Notes are Documents with source_type='note'. This service provides
5note-specific operations while leveraging the Document infrastructure.
6"""
8import atexit
9import hashlib
10import json
11import random
12import re
13import threading
14import time
15import unicodedata
16import uuid
17from concurrent.futures import ThreadPoolExecutor
18from typing import Any, Dict, List, Optional
20from loguru import logger
21from sqlalchemy import func, or_
22from sqlalchemy.exc import IntegrityError
23from sqlalchemy.orm import Session, joinedload
25from ....database.models import (
26 Collection,
27 Document,
28 DocumentChunk,
29 DocumentCollection,
30 NoteChangeType,
31 NoteLink,
32 NoteReference,
33 NoteResearch,
34 NoteVersion,
35 RagDocumentStatus,
36 SourceType,
37)
38from ....database.session_context import get_user_db_session, safe_rollback
39from ....database.thread_local_session import thread_cleanup
40from ...utils import escape_like
43def escape_markdown_link_label(label: str) -> str:
44 """Escape a string so it is safe as the LABEL of a ``[label](url)``
45 markdown link.
47 Provenance/comment notes embed untrusted text — a research query, or a
48 library document's title (which for downloaded documents comes from
49 third-party HTML/PDF metadata) — as the label of an auto-generated
50 link. Without escaping, a label like ``evil](https://attacker/x`` ends
51 the label at its first ``]`` and lets the rest re-target the rendered
52 link at an attacker URL (a phishing/content-integrity issue when the
53 note is later rendered as markdown, and a syntax-injection risk in the
54 PDF/LaTeX/Quarto exporters). Escape the four link-structural
55 metacharacters (backslash first) and neutralise newlines so the label
56 can never break out of the brackets.
57 """
58 return (
59 str(label)
60 .replace("\\", "\\\\")
61 .replace("[", "\\[")
62 .replace("]", "\\]")
63 .replace("(", "\\(")
64 .replace(")", "\\)")
65 .replace("\r", " ")
66 .replace("\n", " ")
67 )
70def _capture_request_db_password(username: str) -> Optional[str]:
71 """Capture the encrypted-DB password from the request thread.
73 Thin wrapper over the shared
74 ``database.session_passwords.capture_request_db_password`` (kept as a
75 module-local name so the many call sites in this file stay unchanged).
76 Needed because ``ThreadPoolExecutor.submit`` does not copy the calling
77 thread's ``ContextVar`` snapshot, so background workers otherwise lose
78 the password and every encrypted-DB write is swallowed.
79 """
80 from ....database.session_passwords import capture_request_db_password
82 return capture_request_db_password(username)
85# Bounded thread pool for AI change-summary generation. summarize_changes is
86# the only call inside update_note that hits a remote LLM, so leaving it on
87# the request thread pays a full round-trip on every content-changing save.
88# We persist the version snapshot synchronously with change_summary=None,
89# then enqueue an UPDATE that fills the summary once the LLM returns. Users
90# see their save reflected in history immediately; the summary populates
91# moments later when they open the version panel.
92_summary_executor: ThreadPoolExecutor | None = None
93_summary_executor_lock = threading.Lock()
96def _get_summary_executor() -> ThreadPoolExecutor:
97 global _summary_executor
98 with _summary_executor_lock:
99 if _summary_executor is None:
100 _summary_executor = ThreadPoolExecutor(
101 max_workers=4,
102 thread_name_prefix="note_summary_",
103 )
104 return _summary_executor
107# Soft cap on pending change-summary tasks. ThreadPoolExecutor's
108# internal queue is unbounded; combined with the per-task (old_content,
109# new_content) capture (up to 50 MB each), a scripted burst of saves
110# could pile up hundreds of MB before the 4 workers drain it. Drop the
111# task with a log line once the queue exceeds the cap — losing the AI
112# change summary is a non-essential degradation; the snapshot row
113# itself already landed at update_note commit time. Per-user write
114# rate-limit (``_notes_write_limit`` at 60/min) provides the primary
115# defence; this is belt-and-suspenders for unexpected hot paths.
116_SUMMARY_QUEUE_SOFT_CAP = 100
119def _submit_summary_task(fn, *args, **kwargs) -> None:
120 """Submit a change-summary task with queue-depth backpressure.
122 On overload, drops the task and logs at WARNING level — better than
123 silent unbounded queue growth. The snapshot row is unaffected;
124 change_summary stays NULL until the next save fills it.
125 """
126 executor = _get_summary_executor()
127 # ThreadPoolExecutor exposes its work queue via the private
128 # ``_work_queue`` attribute. The qsize() reading is approximate
129 # (queue is concurrent-safe but the value can be stale by the
130 # time we read it) but the soft cap intentionally tolerates drift.
131 try:
132 qsize = executor._work_queue.qsize()
133 except Exception:
134 qsize = 0
135 if qsize >= _SUMMARY_QUEUE_SOFT_CAP:
136 logger.warning(
137 "Summary executor queue at {} items (cap {}); dropping task. "
138 "NoteVersion.change_summary will stay NULL for this save.",
139 qsize,
140 _SUMMARY_QUEUE_SOFT_CAP,
141 )
142 return
143 # Wrap in thread_cleanup so the worker clears its thread-local DB
144 # session AND the cached (username, password) entry in
145 # _thread_credentials when it returns. The summary pool's threads are
146 # long-lived (they never die), so cleanup_dead_threads never sweeps
147 # them — without this the plaintext SQLCipher password the worker
148 # captured would sit in the pool thread's dict for the whole process
149 # lifetime (the #4182 credential-hygiene class). Mirrors the idiom
150 # every other executor in the codebase uses (e.g. news_strategy).
151 try:
152 executor.submit(thread_cleanup(fn), *args, **kwargs)
153 except RuntimeError:
154 # The pool was shut down (graceful restart / atexit) between the
155 # _get_summary_executor() read above and this submit. change_summary is
156 # best-effort, so drop it rather than let the RuntimeError bubble up and
157 # 500 a note save whose content already committed durably.
158 logger.warning(
159 "Summary executor shut down before task submit; "
160 "NoteVersion.change_summary will stay NULL for this save."
161 )
164def _shutdown_summary_executor() -> None:
165 # Race fix: the matching getter at _get_summary_executor holds the
166 # lock for its read-modify-write of _summary_executor; the shutdown
167 # path must hold the same lock or a request thread can observe a
168 # half-replaced reference (e.g. submit() into an already-shut-down
169 # executor → RuntimeError) during gunicorn graceful shutdown.
170 global _summary_executor
171 with _summary_executor_lock:
172 if _summary_executor is not None:
173 _summary_executor.shutdown(wait=True)
174 _summary_executor = None
177atexit.register(_shutdown_summary_executor)
180def _populate_change_summary_async(
181 username: str,
182 dbpw: Optional[str],
183 version_id: str,
184 old_content: str,
185 new_content: str,
186) -> None:
187 """Background worker: ask the LLM for a change summary and UPDATE the
188 already-persisted NoteVersion row.
190 ``dbpw`` is the encrypted-DB password captured on the request thread by
191 ``_capture_request_db_password`` and threaded in explicitly because
192 stdlib ``ThreadPoolExecutor`` does not propagate the request's
193 ``ContextVar`` snapshot — without it, ``get_user_db_session`` raises
194 ``DatabaseSessionError`` on every encrypted-DB install. It's named
195 ``dbpw`` rather than ``db_password`` to avoid a gitleaks generic-secret
196 false positive (that rule matches a ``password=<8+ char literal>`` shape);
197 the value is a passthrough credential, never a hardcoded secret.
199 Exceptions are logged but never raised — this runs on a thread pool and
200 a failure here must not crash the worker.
201 """
202 try:
203 from .note_ai_service import NoteAIService
205 # Pass dbpw so the LLM-settings lookup inside summarize_changes ->
206 # _get_llm() can open the encrypted DB on this worker thread (no
207 # request context to resolve the password from).
208 ai_service = NoteAIService(username, dbpw=dbpw)
209 summary = ai_service.summarize_changes(old_content, new_content)
210 if not summary: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true
211 return
213 with get_user_db_session(username, password=dbpw) as session:
214 row = session.query(NoteVersion).filter_by(id=version_id).first()
215 if row is None: 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true
216 logger.debug(
217 "Version {} disappeared before summary landed; skipping",
218 version_id[:8],
219 )
220 return
221 # Fill-only guard: this worker is only ever scheduled for a
222 # freshly inserted snapshot (change_summary=None). Refuse to
223 # overwrite an existing summary so a caller bug (e.g. passing
224 # a dedup-absorbed historical version id) can't corrupt the
225 # audit trail.
226 if row.change_summary is not None:
227 logger.warning(
228 "Version {} already has a change summary; refusing to "
229 "overwrite it",
230 version_id[:8],
231 )
232 return
233 row.change_summary = summary
234 session.commit()
235 logger.debug(
236 "Filled change summary for note version {}", version_id[:8]
237 )
238 except Exception:
239 logger.exception(
240 "Async change-summary worker failed for version {}",
241 version_id[:8] if version_id else "<none>",
242 )
245# Regex for parsing [[wiki-style links]]. The negated class is bounded by
246# MAX_LINK_TEXT_LENGTH so an unclosed ``[[`` doesn't capture multi-MB of
247# body and ship it as a SQL parameter twice (exact + startswith) against
248# the unindexed LOWER(title) column. It also excludes newlines (``\n``): a
249# note title is a single-line String(500), so a ``[[`` whose closing
250# ``]]`` is on a later line is an unclosed bracket, not a link. This
251# mirrors the frontend's ``WIKILINK_TARGET = [^\]\n]+`` in
252# web/static/js/services/formatting.js so parse (which creates NoteLink
253# rows) and render (which draws the link) agree — previously the backend
254# admitted newlines and created links the UI never rendered.
255MAX_LINK_TEXT_LENGTH = 500
256LINK_PATTERN = re.compile(r"\[\[([^\]\n]{1,500})\]\]")
258# Cap on parsed wiki-links per note. Without this, a pathological note
259# (e.g. 50 MB of `[[A]]\n` lines = millions of matches) would spawn
260# millions of NoteLink rows on save and have each one trigger 2-3
261# database queries through ``_resolve_link_internal``. 1000 covers any
262# realistic note (Roam/Obsidian power users typically have <100 links
263# per note); anything beyond is silently truncated and a warning logged
264# rather than failing the save.
265MAX_LINKS_PER_NOTE = 1000
267# Mirrors SQLite's built-in lower(): ASCII-only case folding. Link
268# resolution folds titles in SQL, and the batched exact-title lookup has
269# to match lower(title) values computed BY THE DATABASE back to
270# Python-side link texts — so the Python fold must be the database's.
271# str.lower() folds full Unicode ('Ü' → 'ü') and would diverge.
272_SQLITE_LOWER_TABLE = str.maketrans(
273 "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
274)
277def _fold_title_ascii(text: str) -> str:
278 return (text or "").translate(_SQLITE_LOWER_TABLE)
281# SQLITE_MAX_VARIABLE_NUMBER is 999 on older SQLite builds; chunk IN()
282# lists well below it so a 1000-link note can't overflow the bind-param
283# limit.
284_IN_CLAUSE_CHUNK = 500
286# Caps on per-note many-to-many list params accepted by the route
287# layer. Mirror MAX_TAGS_PER_NOTE in spirit: bound how many DocumentCollection
288# / NoteResearch rows a single create or reorder request can spawn.
289MAX_COLLECTIONS_PER_NOTE = 50
290MAX_RESEARCH_PER_NOTE = 200
292# Mirrors BaseExporter.MAX_CONTENT_SIZE at exporters/base.py:63. Kept inline
293# rather than imported because exporters/__init__.py unconditionally pulls in
294# weasyprint via all 5 exporter modules — importing BaseExporter would drag a
295# heavy dependency into note CRUD. Extract to a dep-light constants module if
296# sharing ever becomes worthwhile.
297NOTE_CONTENT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
299# Cap on version-history rows per note. Auto-save can otherwise grow
300# note_versions unboundedly — a near-50 MB note with small changes x 100 saves
301# = ~5 GB. Pure FIFO: simplest rule with useful value; the `initial` version
302# can be pruned, but the user still has 100 recent snapshots to fall back on.
303MAX_VERSIONS_PER_NOTE = 100
304# Audit bookends (PRE_RESTORE / RESTORE) are excluded from the normal version
305# prune so the "restored here" trail survives heavy editing — but they still
306# need a ceiling, or repeated restores grow note_versions without bound. Keep
307# the most recent N bookends.
308MAX_BOOKEND_VERSIONS = 40
310# PRE_RESTORE / RESTORE are the audit bookends excluded from version-dedup
311# (their hashes are salted) and from the prune pool (the "restored here" trail
312# survives heavy editing).
313_AUDIT_BOOKEND_TYPES = (
314 NoteChangeType.PRE_RESTORE.value,
315 NoteChangeType.RESTORE.value,
316)
319def _assert_content_size(content: str) -> None:
320 """Raise ValueError if content isn't a string or exceeds
321 NOTE_CONTENT_MAX_BYTES.
323 The type check matters for direct API callers: a truthy non-string
324 body value (e.g. ``{"content": 123}``) passes the route layer's
325 ``if not content`` guard, and without this check ``.encode()`` raised
326 an AttributeError that surfaced as an opaque 500 instead of the 400
327 the routes' ``except ValueError`` mapping produces.
328 """
329 if not isinstance(content, str):
330 raise ValueError("content must be a string")
331 if content and len(content.encode("utf-8")) > NOTE_CONTENT_MAX_BYTES:
332 raise ValueError(
333 f"Note content exceeds maximum size "
334 f"({NOTE_CONTENT_MAX_BYTES // (1024 * 1024)} MB)"
335 )
338# Mirrors MAX_TAG_LENGTH in note-detail.js. The frontend caps the input
339# field, but direct API callers can still submit arbitrary tag strings.
340# Backend validation is the only stop against a 10MB tag landing in the
341# JSON ``tags`` column.
342MAX_TAG_LENGTH = 50
344# Cap the number of tags per note. Without this, ``_validate_tags`` accepts
345# an unbounded list — the 50-char per-tag limit then permits multi-MB of
346# tag JSON per note, and snapshots store the tag list per version, so the
347# 100-version cap multiplies the bloat.
348MAX_TAGS_PER_NOTE = 50
351def _validate_tags(tags: Optional[List[str]]) -> None:
352 """Raise ValueError if ``tags`` isn't a list of short strings.
354 None passes (callers may pass None to mean "don't update tags"). An
355 empty list passes. Anything else: must be a list, every entry must
356 be a string, and each string must be ``<= MAX_TAG_LENGTH`` chars.
357 """
358 if tags is None:
359 return
360 if not isinstance(tags, list):
361 raise ValueError("tags must be a list")
362 if len(tags) > MAX_TAGS_PER_NOTE:
363 raise ValueError(f"too many tags (max {MAX_TAGS_PER_NOTE})")
364 for tag in tags:
365 if not isinstance(tag, str):
366 raise ValueError("each tag must be a string")
367 if len(tag) > MAX_TAG_LENGTH:
368 raise ValueError(
369 f"tag exceeds maximum length ({MAX_TAG_LENGTH} chars)"
370 )
373# Cap on note title length. ``Document.title`` is a plain ``Text`` column
374# (unbounded), but the title is interpolated raw into LLM synthesis prompts
375# (note_ai_service.py:729) and into ``LOWER(title) == ?`` wiki-link lookups
376# (note_service._resolve_link_internal) on every save. Without a cap a
377# direct API caller could send a 50 MB title and DoS both paths.
378MAX_TITLE_LENGTH = 500
381def _validate_title(title: Optional[str]) -> None:
382 """Raise ValueError if ``title`` isn't a short non-blank string.
384 None passes (callers may pass None to "don't update title"). Empty
385 string passes (the route layer's own ``if not title`` check still
386 rejects it on create). Whitespace-only strings (``" "``,
387 ``"\\t\\n"``) are explicitly rejected so the slug generator does not
388 produce an empty slug and the wiki-link resolver doesn't match an
389 invisible title via ``[[ ]]``. Anything else: must be a string and
390 at most ``MAX_TITLE_LENGTH`` characters.
391 """
392 if title is None:
393 return
394 if not isinstance(title, str): 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 raise ValueError("title must be a string")
396 # Reject whitespace-only titles. Empty string falls through to the
397 # route-layer guard; whitespace-only is truthy and would slip past
398 # ``if not title`` checks.
399 if title and not title.strip():
400 raise ValueError("title must not be blank or whitespace-only")
401 if len(title) > MAX_TITLE_LENGTH:
402 raise ValueError(
403 f"title exceeds maximum length ({MAX_TITLE_LENGTH} chars)"
404 )
407class NoteService:
408 """Service for managing notes (Documents with source_type='note')."""
410 DEFAULT_NOTES_COLLECTION_NAME = "Notes"
411 SOURCE_TYPE_NAME = "note"
413 def __init__(self, username: str):
414 """Initialize note service for a user."""
415 self.username = username
417 @staticmethod
418 def _generate_slug(title: str) -> str:
419 """Generate a URL-friendly slug from a title.
421 Folds Latin accents to ASCII via stdlib NFKD normalization
422 (``café`` → ``cafe``). Scripts with no ASCII form (CJK, Cyrillic,
423 Arabic) and emoji drop out, so a title that reduces to nothing falls
424 back to the ``"note"`` placeholder. A real transliterator is avoided
425 on purpose: the licensed options (text-unidecode / python-slugify)
426 are copyleft/GPL, and the slug is a non-critical convenience field.
427 """
428 ascii_title = (
429 unicodedata.normalize("NFKD", title or "")
430 .encode("ascii", "ignore")
431 .decode("ascii")
432 )
433 slug = ascii_title.lower()
434 slug = re.sub(r"\s+", "-", slug)
435 slug = re.sub(r"[^a-z0-9\-]", "", slug)
436 slug = re.sub(r"-+", "-", slug)
437 slug = slug.strip("-")
438 return (slug or "note")[:500]
440 @staticmethod
441 def _compute_content_hash(content: str) -> str:
442 """Compute SHA-256 hash of content."""
443 return hashlib.sha256(content.encode("utf-8")).hexdigest()
445 @staticmethod
446 def _compute_version_hash(
447 title: Optional[str],
448 content: Optional[str],
449 tags: Optional[List[str]],
450 ) -> str:
451 """Compute the dedup hash for a NoteVersion snapshot.
453 JSON-encodes the whole ``[title, content, sorted-tags]`` triple and
454 hashes that, so title-only / content-only / tag-only edits each get a
455 distinct hash and aren't silently deduped against an existing snapshot.
456 Tags are sorted first so reordering alone doesn't churn the hash.
458 Why JSON the WHOLE triple rather than a NUL-joined string: title and
459 content can each contain a literal NUL byte — a JSON ``\\u0000`` in the
460 request body decodes to ``\\x00`` and survives SQLite TEXT storage
461 untouched (SQLite is length-prefixed, not NUL-terminated), and nothing
462 validates it away — so a NUL separator is NOT injection-proof. Pre-fix,
463 ``title="MyTitle", content="Hello\\x00World"`` and
464 ``title="MyTitle\\x00Hello", content="World"`` produced the identical
465 NUL-joined payload and hash, silently dedup-absorbing a genuinely
466 different save out of version history. JSON escapes every control
467 character (NUL included) as ``\\uXXXX`` and quotes/escapes each element,
468 making the encoding injective across all three fields.
470 Hash-format note: this serialization differs from older builds', so the
471 hash changes for previously-written snapshots. The only consumer is the
472 (document_id, content_hash) dedup check, so the sole effect is a
473 one-time extra version row the first time a note is re-saved in a state
474 that already has an old-format snapshot — benign.
475 """
476 title_s = "" if title is None else title
477 content_s = "" if content is None else content
478 sorted_tags = sorted(t for t in (tags or []) if isinstance(t, str))
479 payload = json.dumps(
480 [title_s, content_s, sorted_tags], ensure_ascii=False
481 )
482 return hashlib.sha256(payload.encode("utf-8")).hexdigest()
484 def _get_note_source_type_id(self, session: Session) -> str:
485 """Get the source_type ID for notes, creating it if needed."""
486 from . import get_note_source_type_id
488 source_type_id = get_note_source_type_id(session)
489 if source_type_id: 489 ↛ 493line 489 didn't jump to line 493 because the condition on line 489 was always true
490 return source_type_id
492 # Create it if it doesn't exist (shouldn't happen normally)
493 source_type = SourceType(
494 id=str(uuid.uuid4()),
495 name=self.SOURCE_TYPE_NAME,
496 display_name="Note",
497 description="User-created notes with AI-enhanced features",
498 icon="sticky-note",
499 )
500 session.add(source_type)
501 session.flush()
502 return source_type.id
504 def _get_or_create_notes_collection(self, session: Session) -> str:
505 """Get or create the default Notes collection.
507 Serialised by the per-user init lock to prevent two concurrent
508 first-note creations (e.g. two browser tabs both creating their
509 first note simultaneously) from each seeing no ``collection_type='notes'``
510 row and inserting a duplicate. Matches the locking pattern used
511 by ``ensure_default_library_collection`` and
512 ``ensure_research_history_collection`` in ``database/library_init.py``;
513 there is no UNIQUE constraint at the schema level that would
514 catch the race otherwise.
516 The create path COMMITS inside the lock, for two reasons (both
517 matching the ``library_init`` reference pattern):
519 1. Lock scope: a flush-only insert leaves the row invisible to
520 other sessions until the caller commits — after the lock is
521 released — so a second request entering the lock in that window
522 would still see no row and insert a duplicate.
523 2. Read-only callers: routes like ask-context reach this via
524 ``get_notes_collection_id`` and never commit; the request
525 teardown ROLLS BACK the shared session, discarding the row
526 after its id was already returned to the client.
528 Because the commit flushes the caller's whole session, callers
529 must resolve the collection BEFORE staging unrelated writes on
530 the same session (create_note does).
531 """
532 from ....database.library_init import _get_user_init_lock
534 with _get_user_init_lock(self.username):
535 collection = (
536 session.query(Collection)
537 .filter_by(collection_type="notes")
538 .first()
539 )
541 if collection:
542 return collection.id
544 collection_id = str(uuid.uuid4())
545 collection = Collection(
546 id=collection_id,
547 name=self.DEFAULT_NOTES_COLLECTION_NAME,
548 description="Default collection for notes",
549 collection_type="notes",
550 )
551 session.add(collection)
552 try:
553 session.commit()
554 except Exception:
555 session.rollback()
556 raise
557 logger.info("Created default Notes collection: {}", collection_id)
558 return collection_id
560 def get_notes_collection_id(self) -> Optional[str]:
561 """Get the default Notes collection ID.
563 Returns the collection ID or None if it cannot be determined.
564 """
565 try:
566 with get_user_db_session(self.username) as session:
567 return self._get_or_create_notes_collection(session)
568 except Exception:
569 logger.exception("Error getting notes collection ID")
570 return None
572 def get_notes_index_status(self) -> Dict[str, Any]:
573 """Pre-flight context for "Ask your notes": the Notes collection id,
574 the total note count, and how many notes are indexed into it.
576 The UI uses this to warn when there is nothing to search — no notes
577 at all, or notes that haven't been embedded yet — before kicking off
578 a research run pinned to the Notes collection.
579 """
580 collection_id = self.get_notes_collection_id()
581 note_count = self.count_notes()
582 indexed_count = 0
583 if collection_id: 583 ↛ 590line 583 didn't jump to line 590 because the condition on line 583 was always true
584 with get_user_db_session(self.username) as session:
585 indexed_count = (
586 session.query(DocumentCollection)
587 .filter_by(collection_id=collection_id, indexed=True)
588 .count()
589 )
590 return {
591 "collection_id": collection_id,
592 "note_count": note_count,
593 "indexed_count": indexed_count,
594 }
596 def _is_note(self, session: Session, document: Document) -> bool:
597 """Check if a document is a note."""
598 source_type = (
599 session.query(SourceType)
600 .filter_by(id=document.source_type_id)
601 .first()
602 )
603 return source_type and source_type.name == self.SOURCE_TYPE_NAME
605 def _get_note_in_session(
606 self, session: Session, note_id: str
607 ) -> Optional[Document]:
608 """Fetch a Document by id and return it only if it is a note.
610 Centralizes the fetch + ``_is_note`` guard prelude that every
611 mutating note method hand-copied — and which was half-copied as a
612 bug on the collection helpers (they originally skipped the
613 ``_is_note`` check, letting a non-note Document acquire note rows).
614 Returns ``None`` when the id doesn't exist or the Document isn't a
615 note; callers map that to their own bail value.
617 Still routes through ``self._is_note`` so the existing
618 ``patch.object(NoteService, '_is_note')`` monkeypatches keep firing.
619 """
620 document = session.query(Document).filter_by(id=note_id).first()
621 if document is None or not self._is_note(session, document):
622 return None
623 return document
625 @staticmethod
626 def _escape_like(text: str) -> str:
627 """Escape LIKE/ILIKE wildcards so user input matches literally.
629 Thin alias for the shared :func:`escape_like` so every notes query
630 path uses the one canonical escaping rule.
631 """
632 return escape_like(text)
634 @staticmethod
635 def _rollback_quietly(session) -> None:
636 """Roll back a session after a failed flush/commit, swallowing any
637 rollback error. The per-user request session is shared and is NOT
638 rolled back by get_user_db_session on exception, so callers that catch
639 IntegrityError (and want to retry or surface a clean error) must roll
640 back explicitly or the next statement reuses a poisoned session.
642 Keeps the None-guard (call sites pass ``locals().get("session")``,
643 which may be None) and delegates the actual rollback/log to the
644 shared :func:`safe_rollback` helper.
645 """
646 if session is None: 646 ↛ 647line 646 didn't jump to line 647 because the condition on line 646 was never true
647 return
648 safe_rollback(session, "note_service")
650 def create_note(
651 self,
652 title: str,
653 content: str,
654 tags: Optional[List[str]] = None,
655 collection_ids: Optional[List[str]] = None,
656 ) -> str:
657 """
658 Create a new note (stored as a Document).
660 Args:
661 title: Note title
662 content: Note content (markdown)
663 tags: Optional list of tags
664 collection_ids: Optional list of collection IDs
666 Returns:
667 The created note/document ID
668 """
669 _validate_title(title)
670 _assert_content_size(content)
671 _validate_tags(tags)
672 if collection_ids is not None:
673 if not isinstance(collection_ids, list): 673 ↛ 674line 673 didn't jump to line 674 because the condition on line 673 was never true
674 raise ValueError("collection_ids must be a list")
675 if len(collection_ids) > MAX_COLLECTIONS_PER_NOTE:
676 raise ValueError(
677 f"too many collection_ids (max {MAX_COLLECTIONS_PER_NOTE})"
678 )
679 # Element type check: an unhashable element (dict/list) would
680 # otherwise raise TypeError in the de-dupe set comprehension
681 # below — an opaque 500 instead of the routes' ValueError→400.
682 for cid in collection_ids:
683 if not isinstance(cid, str):
684 raise ValueError("each collection_id must be a string")
685 try:
686 with get_user_db_session(self.username) as session:
687 note_id = str(uuid.uuid4())
688 source_type_id = self._get_note_source_type_id(session)
689 content_hash = self._compute_content_hash(
690 f"{note_id}:{content}"
691 )
693 # Resolve the default Notes collection BEFORE staging the
694 # note: the get-or-create commits when it has to create the
695 # collection (see its docstring), and committing after
696 # session.add(document) would break this method's
697 # one-transaction atomicity for the note itself. A lazily
698 # created SourceType (flushed above) riding along on that
699 # commit is fine — it's the same kind of idempotent init
700 # row as the collection.
701 default_collection_id = self._get_or_create_notes_collection(
702 session
703 )
705 # Create Document with source_type='note'
706 document = Document(
707 id=note_id,
708 source_type_id=source_type_id,
709 document_hash=content_hash,
710 file_size=len(content.encode("utf-8")),
711 file_type="note",
712 title=title,
713 text_content=content,
714 tags=tags or [],
715 # DB column stays `favorite` (the flag is shared with the
716 # Document model); the notes API/UI vocabulary for it is
717 # `pinned` (see the `"pinned": document.favorite` response
718 # mapping). One column, two names by design.
719 favorite=False,
720 character_count=len(content),
721 word_count=len(content.split()),
722 )
723 session.add(document)
725 # Add to default Notes collection
726 doc_coll = DocumentCollection(
727 document_id=note_id,
728 collection_id=default_collection_id,
729 indexed=False,
730 )
731 session.add(doc_coll)
733 # Add to additional collections. De-dupe and skip ids that
734 # don't exist: a repeated id would violate the
735 # (document_id, collection_id) UNIQUE and a bogus id would
736 # FK-violate — either one aborting the entire note creation
737 # with an opaque error.
738 if collection_ids:
739 requested = {cid for cid in collection_ids if cid} - {
740 default_collection_id
741 }
742 valid_ids = {
743 row[0]
744 for row in session.query(Collection.id).filter(
745 Collection.id.in_(requested)
746 )
747 }
748 for coll_id in valid_ids:
749 session.add(
750 DocumentCollection(
751 document_id=note_id,
752 collection_id=coll_id,
753 indexed=False,
754 )
755 )
757 # Atomicity: Document + DocumentCollection + INITIAL
758 # version snapshot + outgoing links all land in one
759 # transaction. Pre-fix, the document committed first and
760 # the snapshot + link reparse ran in fresh sessions, so a
761 # crash between writes left a note with no INITIAL audit
762 # row and no parsed [[wiki-links]] — same shape as the
763 # bug that update_note already fixed.
764 self._create_version_snapshot_in_session(
765 session,
766 note_id=note_id,
767 title=title,
768 content=content,
769 tags=tags,
770 change_type=NoteChangeType.INITIAL.value,
771 change_summary="Initial version",
772 )
773 self._parse_and_update_links_in_session(
774 session, note_id, content
775 )
777 try:
778 session.commit()
779 except Exception:
780 session.rollback()
781 raise
783 # Don't log the title — note titles can carry sensitive
784 # content (e.g. medical, personal). Note id alone is
785 # enough to correlate with surrounding events.
786 logger.info("Created note {}", note_id)
788 return note_id
790 except Exception:
791 # Same atomicity guarantee as update_note: the cached
792 # request/thread session does NOT auto-rollback. The
793 # in-session helpers (snapshot, link reparse) call
794 # session.flush() at the end, so a downstream failure leaves
795 # the document INSERT durably flushed without ever committing
796 # properly. Rollback explicitly so create_note is atomic.
797 # locals().get avoids NameError when the failure happens before
798 # `with ... as session` binds; _rollback_quietly no-ops on None.
799 self._rollback_quietly(locals().get("session"))
800 logger.exception("Error creating note")
801 raise
803 def note_exists(self, note_id: str) -> bool:
804 """Cheap existence + is-note guard for route-level 404 checks.
806 Projects only ``Document.id`` filtered by the note source type —
807 no ``text_content`` hydration (a plain non-deferred column, up to
808 50 MB per row via ``get_note``) and none of ``get_note``'s
809 aggregate side queries. Use this wherever the caller only needs
810 to know the note is real; ``get_note`` stays for handlers that
811 consume the payload.
812 """
813 if not note_id: 813 ↛ 814line 813 didn't jump to line 814 because the condition on line 813 was never true
814 return False
815 with get_user_db_session(self.username) as session:
816 note_source_type_id = self._get_note_source_type_id(session)
817 if note_source_type_id is None: 817 ↛ 818line 817 didn't jump to line 818 because the condition on line 817 was never true
818 return False
819 return (
820 session.query(Document.id)
821 .filter(
822 Document.id == note_id,
823 Document.source_type_id == note_source_type_id,
824 )
825 .first()
826 is not None
827 )
829 def get_note(self, note_id: str) -> Optional[Dict[str, Any]]:
830 """Get a note by ID.
832 The response includes an ``outgoing_links`` map (link_text →
833 target_id, target_title) so the frontend can resolve
834 ``[[wiki-links]]`` to the correct target Document at render
835 time, regardless of whether the target has since been renamed.
836 Pre-fix the frontend resolved by title text alone via
837 ``navigateToNoteByTitle``, so renaming a linked note silently
838 broke every existing ``[[OldTitle]]`` in other notes' content.
839 """
840 try:
841 with get_user_db_session(self.username) as session:
842 document = self._get_note_in_session(session, note_id)
843 if document is None:
844 return None
846 result = self._document_to_note_dict(document, session)
848 # Outgoing-link resolution table for the frontend wiki-link
849 # renderer. Keep this lean (only the link_text and the
850 # target_id/title) — the full backlinks payload is served
851 # by GET /api/notes/<id>/outgoing-links on demand.
852 outgoing_links = (
853 session.query(
854 NoteLink.link_text,
855 Document.id,
856 Document.title,
857 )
858 .join(NoteLink, Document.id == NoteLink.target_document_id)
859 .filter(NoteLink.source_document_id == note_id)
860 .all()
861 )
862 result["outgoing_links"] = [
863 {
864 "link_text": row.link_text,
865 "target_id": row.id,
866 "target_title": row.title,
867 }
868 for row in outgoing_links
869 ]
870 return result
872 except Exception:
873 logger.exception("Error getting note {}", note_id)
874 raise
876 def update_note(
877 self,
878 note_id: str,
879 title: Optional[str] = None,
880 content: Optional[str] = None,
881 tags: Optional[List[str]] = None,
882 favorite: Optional[bool] = None,
883 _link_overrides: Optional[Dict[str, str]] = None,
884 _append_link_title: Optional[str] = None,
885 ) -> bool:
886 """Update a note.
888 Naming note: ``favorite`` matches ``Document.favorite`` (the column
889 backing this state). The user-facing UI vocabulary is "pin"; the
890 wire format keeps ``pinned`` for that reason and the route layer
891 translates. Don't rename the kwarg back without also updating the
892 Document column.
894 ``_append_link_title`` (internal, used by ``accept_suggested_link``):
895 when set, the new body is derived from the note's CURRENT content —
896 loaded fresh inside this method's write transaction — by appending a
897 ``[[title]]`` wiki-link, rather than being passed in precomputed by the
898 caller. This keeps the read-modify-write atomic and closes the
899 lost-update window where a concurrent save committed after the caller's
900 precondition read would otherwise be clobbered. Mutually exclusive with
901 ``content``.
902 """
903 _validate_title(title)
904 # Empty title is allowed on create's route-level guard, but here on
905 # update an explicit empty string would erase a previously valid
906 # title with no semantic meaning. Reject it.
907 if title is not None and not title:
908 raise ValueError("title must not be empty")
909 if content is not None:
910 _assert_content_size(content)
911 _validate_tags(tags)
912 try:
913 with get_user_db_session(self.username) as session:
914 document = self._get_note_in_session(session, note_id)
915 if document is None:
916 return False
918 # Append-mode (accept_suggested_link): derive the new body from
919 # the FRESHLY loaded row inside this write transaction, rather
920 # than from a value the caller read in an earlier, now-closed
921 # session. Doing the read-modify-write in one transaction closes
922 # the lost-update window where a concurrent save committed
923 # between the caller's precondition read and this write would
924 # otherwise be silently clobbered.
925 _append_reparse_only = False
926 if _append_link_title is not None:
927 current = document.text_content or ""
928 # If the exact wiki-link text is already in the body, don't
929 # DUPLICATE the line — but STILL reparse links so the target
930 # NoteLink is created/updated. This covers two cases:
931 # (a) a concurrent second accept of the SAME suggestion
932 # (double-submit / two tabs / a retry after a
933 # processed-but-timed-out response) whose first accept
934 # already appended the line and created the NoteLink —
935 # the reparse is then an idempotent no-op; and
936 # (b) accepting a suggestion for a link the user had TYPED
937 # while it was still unresolved (no NoteLink row exists
938 # yet, e.g. the target note was created later). Skipping
939 # the reparse here would make the user's explicit accept
940 # a silent no-op that never creates the link.
941 if f"[[{_append_link_title}]]" in current:
942 content = None
943 _append_reparse_only = True
944 else:
945 content = (
946 current.rstrip() + f"\n\n[[{_append_link_title}]]\n"
947 )
948 _assert_content_size(content)
950 content_changed = (
951 content is not None and content != document.text_content
952 )
953 title_changed = title is not None and title != document.title
954 # Tags are part of the version state (they participate in
955 # _compute_version_hash). A tag-only edit must produce a
956 # snapshot — otherwise the hash improvement is dead code on
957 # this path and the user's version history silently misses
958 # tag-only changes.
959 #
960 # Use sorted() so reordering the same tag set
961 # (e.g. ["b","a"] vs ["a","b"]) doesn't flip the trigger.
962 # _compute_version_hash already sorts before hashing, so an
963 # order-only "change" would just create a dedup-absorbed
964 # snapshot trigger, wasting work; aligning the comparison
965 # keeps trigger and hash consistent.
966 tags_changed = tags is not None and sorted(
967 t for t in tags if isinstance(t, str)
968 ) != sorted(
969 t for t in (document.tags or []) if isinstance(t, str)
970 )
971 old_content = document.text_content
973 if title is not None:
974 document.title = title
975 if content is not None:
976 document.text_content = content
977 document.character_count = len(content)
978 document.word_count = len(content.split())
979 document.document_hash = self._compute_content_hash(
980 f"{note_id}:{content}"
981 )
982 document.file_size = len(content.encode("utf-8"))
983 if tags is not None:
984 document.tags = tags
985 if favorite is not None:
986 document.favorite = favorite
988 # Atomicity: document update + version snapshot + link
989 # rewrite all share the same transaction. Pre-fix, each
990 # ran in its own session, so a crash between them left a
991 # committed content change with no MANUAL_SAVE row
992 # or with zero outgoing links. The AI change-summary
993 # generation still runs off-thread — it's enqueued AFTER
994 # commit so a failed transaction won't leave a stranded
995 # background task.
996 version_id: Optional[str] = None
997 version_created = False
998 # (0, 0) means "prune wasn't run this call" as well as "ran
999 # but nothing was over cap" — both cases skip the post-commit
1000 # log the same way, so a single sentinel default covers both.
1001 prune_counts: "tuple[int, int]" = (0, 0)
1002 # The version-snapshot INSERT is flushed inside
1003 # _create_version_snapshot_in_session, so a concurrent
1004 # identical-content save's UNIQUE(uix_note_version_content)
1005 # collision can fire at THAT flush, not only at the commit
1006 # below. Wrap the whole snapshot + link-reparse + commit block
1007 # so it is caught and recovered in one place — previously the
1008 # recovery guarded only commit(), so a flush-time collision
1009 # 500'd a benign concurrent save (the recovery was dead code).
1010 try:
1011 if content_changed or title_changed or tags_changed:
1012 (
1013 version_id,
1014 version_created,
1015 ) = self._create_version_snapshot_in_session(
1016 session,
1017 note_id=note_id,
1018 title=document.title,
1019 content=document.text_content,
1020 tags=document.tags,
1021 change_type=NoteChangeType.MANUAL_SAVE.value,
1022 change_summary=None,
1023 )
1024 # Counts are only bound here, not logged — the
1025 # delete is merely flushed, and this whole block can
1026 # still roll back below (IntegrityError recovery) or
1027 # in the outer handler. Logging happens after
1028 # session.commit() succeeds, so the log never claims
1029 # a prune a rollback undid.
1030 prune_counts = self._prune_versions_in_session(
1031 session, note_id
1032 )
1034 if content_changed or _append_reparse_only:
1035 # In append-reparse-only mode `content` is None (the body
1036 # already held the link text); reparse the CURRENT body
1037 # so the override still binds the NoteLink to the target.
1038 self._parse_and_update_links_in_session(
1039 session,
1040 note_id,
1041 content
1042 if content is not None
1043 else (document.text_content or ""),
1044 link_overrides=_link_overrides,
1045 )
1047 # Stale-vector fix: when the note's content or title
1048 # changes, mark its embeddings stale (both indexed-state
1049 # sources) so the auto-index worker re-embeds AND the RAG
1050 # status report doesn't keep showing it as indexed. See
1051 # _mark_note_stale_for_reindex_in_session.
1052 if content_changed or title_changed:
1053 self._mark_note_stale_for_reindex_in_session(
1054 session, note_id
1055 )
1057 session.commit()
1058 # Only reachable once the transaction holding the prune
1059 # deletes has actually committed — see
1060 # _prune_versions_in_session for why this can't log
1061 # from inside the prune itself.
1062 self._log_version_prune(note_id, prune_counts)
1063 except IntegrityError as exc:
1064 # Concurrency: two same-note saves carrying byte-identical
1065 # content both pass the in-session dedup pre-check in
1066 # _create_version_snapshot_in_session (neither sees the
1067 # other's uncommitted row), then race to INSERT the same
1068 # (document_id, content_hash). The winner commits; the
1069 # loser hits UNIQUE(uix_note_version_content) — at the
1070 # snapshot flush above or at the commit. That
1071 # collision means the identical version already exists and
1072 # the winner already persisted the identical document
1073 # content — so the user's save IS reflected. Recover
1074 # (rollback + return success) instead of 500'ing.
1075 #
1076 # Discriminator: only the version-dedup constraint carries
1077 # "content_hash" (SQLite reports the offending columns,
1078 # Postgres the constraint name uix_note_version_content —
1079 # both contain "content_hash"). Any other IntegrityError
1080 # (e.g. a NoteLink uix_note_link violation from the link
1081 # reparse) must still surface, so re-raise it.
1082 self._rollback_quietly(session)
1083 err = (
1084 str(exc.orig).lower() if exc.orig else str(exc).lower()
1085 )
1086 if "content_hash" not in err:
1087 raise
1088 logger.debug(
1089 "Concurrent identical-content save of note {} "
1090 "collided on the version-dedup UNIQUE; the existing "
1091 "version already represents this content. Treating as "
1092 "success.",
1093 note_id,
1094 )
1095 # The winner persisted the identical VERSIONED state
1096 # (title/content/tags — the fields in the version
1097 # hash), but ``favorite`` is not hashed: a loser that
1098 # also carried a pin change had it rolled back with
1099 # the rest of its transaction, so returning success
1100 # here would silently drop the pin. Re-apply it in a
1101 # fresh transaction — a favorite-only UPDATE cannot
1102 # collide on the version constraint. A failure here
1103 # propagates (outer handler) rather than reporting a
1104 # success the DB doesn't reflect.
1105 if favorite is not None:
1106 refetched = self._get_note_in_session(session, note_id)
1107 if ( 1107 ↛ 1113line 1107 didn't jump to line 1113 because the condition on line 1107 was always true
1108 refetched is not None
1109 and refetched.favorite != favorite
1110 ):
1111 refetched.favorite = favorite
1112 session.commit()
1113 return True
1115 logger.info("Updated note {}", note_id)
1117 # Post-commit: enqueue the LLM change-summary worker. The
1118 # snapshot row already exists with change_summary=None; the
1119 # worker UPDATEs it when the LLM returns. Only schedule it
1120 # for a genuinely NEW row: when the dedup check absorbed
1121 # the snapshot into an existing version (e.g. the user
1122 # reverted to an exact prior state), version_id points at a
1123 # HISTORICAL row whose change_summary the worker would
1124 # overwrite with a description of this much later edit.
1125 if (
1126 version_id
1127 and version_created
1128 and content_changed
1129 and content is not None
1130 ):
1131 # NB: intentionally NOT gated on `old_content` being
1132 # truthy. A first-content edit of a previously blank/NULL
1133 # note (old_content "" or None) is still worth summarizing
1134 # — summarize_changes handles it cheaply (`if not
1135 # old_content: return "Note created"`, no LLM call). Gating
1136 # on truthy old_content left that MANUAL_SAVE version's
1137 # change_summary NULL forever.
1138 # Capture the DB password from the request thread now;
1139 # the worker runs on a stdlib ThreadPoolExecutor which
1140 # does NOT copy ContextVar state, so the worker cannot
1141 # locate the password itself.
1142 dbpw = _capture_request_db_password(self.username)
1143 if dbpw is None:
1144 # Without the password the worker can't open the
1145 # encrypted DB to write the summary. Pre-fix this
1146 # was silently dropped by the worker itself; log
1147 # explicitly and skip the submit so operators can
1148 # see WHY summaries are missing.
1149 logger.warning(
1150 "Skipping AI change-summary for version {}: "
1151 "DB password not available in request thread.",
1152 version_id,
1153 )
1154 else:
1155 _submit_summary_task(
1156 _populate_change_summary_async,
1157 self.username,
1158 dbpw,
1159 version_id,
1160 old_content,
1161 content,
1162 )
1164 return True
1166 except Exception:
1167 # ``get_user_db_session``'s context manager does NOT rollback
1168 # on exception (it returns a cached request- or thread-scoped
1169 # session whose lifecycle is managed elsewhere). Helpers like
1170 # ``_create_version_snapshot_in_session`` call ``session.flush()``
1171 # at the end so a downstream failure (e.g. in link reparse)
1172 # would otherwise leave the document UPDATE durably flushed
1173 # without ever committing properly. Rollback explicitly here
1174 # so the atomicity guarantee holds.
1175 # ``session`` is bound iff we entered the with block; locals().get
1176 # avoids NameError when the failure happens before then, and
1177 # _rollback_quietly no-ops on None.
1178 self._rollback_quietly(locals().get("session"))
1179 logger.exception("Error updating note {}", note_id)
1180 raise
1182 def delete_note(self, note_id: str) -> bool:
1183 """Delete a note.
1185 Before the row is removed we collect the set of collections this note
1186 was indexed into, because the ``Document → DocumentCollection``
1187 cascade fires on delete and would otherwise hide them. After the
1188 delete commits we purge the corresponding ``DocumentChunk`` rows for
1189 each indexed collection via
1190 ``LibraryRAGService.purge_document_chunks`` — NOT
1191 ``remove_document_from_rag``, which looks the document up by its
1192 (now cascade-deleted) ``DocumentCollection`` join row and would
1193 silently no-op.
1195 Both halves of the RAG state are purged: the FAISS vectors first
1196 (``purge_document_vectors`` — replace-on-reindex can never fire
1197 again for a deleted document id, and the shared search flows
1198 rehydrate snippets straight from the ``DocumentChunk`` rows in the
1199 (encrypted) DB by id, with no Document-existence check, so
1200 lingering vectors would serve the deleted note's text
1201 indefinitely), then the ``DocumentChunk`` rows
1202 (``purge_document_chunks``). The ghost-hit filter in
1203 ``NoteAIService.semantic_search`` / ``find_similar_passages``
1204 remains as the safety net for cleanup failures and pre-fix
1205 leftovers.
1206 """
1207 try:
1208 indexed_collection_ids: List[str] = []
1209 with get_user_db_session(self.username) as session:
1210 document = self._get_note_in_session(session, note_id)
1211 if document is None:
1212 return False
1214 # Capture (before the cascade destroys them) every collection
1215 # where this note might have vectors to purge: the UNION of
1216 # DocumentCollection.indexed=True AND collections that actually
1217 # have chunk rows. The chunk-row half is essential — an edited
1218 # note has indexed=False during the edit->reindex window while
1219 # its pre-edit chunks/vectors still exist, so gating on the
1220 # indexed flag alone would strand orphaned vectors.
1221 indexed_rows = (
1222 session.query(DocumentCollection.collection_id)
1223 .filter_by(document_id=note_id, indexed=True)
1224 .all()
1225 )
1226 chunk_rows = (
1227 session.query(DocumentChunk.collection_name)
1228 .filter_by(source_type="document", source_id=note_id)
1229 .distinct()
1230 .all()
1231 )
1232 indexed_collection_ids = list(
1233 {row[0] for row in indexed_rows}
1234 | {
1235 row[0].removeprefix("collection_")
1236 for row in chunk_rows
1237 if row[0]
1238 }
1239 )
1241 session.delete(document)
1242 session.commit()
1243 logger.info("Deleted note {}", note_id)
1245 # Best-effort FAISS / chunk cleanup. Failures must not surface
1246 # to the user (the DB delete already committed) but should be
1247 # logged for ops visibility.
1248 if indexed_collection_ids:
1249 dbpw = _capture_request_db_password(self.username)
1250 from ...services.rag_service_factory import get_rag_service
1252 for collection_id in indexed_collection_ids:
1253 try:
1254 # A FRESH service per collection, built via the
1255 # factory so it resolves THAT collection's stored
1256 # embedding model/provider (get_rag_service reads
1257 # Collection settings; a hardcoded-default
1258 # LibraryRAGService would compute the wrong index
1259 # hash and silently purge nothing for any
1260 # non-default-embedding collection). Per-collection
1261 # instances also avoid one collection's cached
1262 # ``rag_index_record`` / loaded vector index
1263 # carrying over into another collection's
1264 # ``purge_document_vectors`` call.
1265 with get_rag_service(
1266 self.username,
1267 collection_id=collection_id,
1268 db_password=dbpw,
1269 ) as rag:
1270 # Vectors BEFORE chunk rows: purge_document_vectors
1271 # -> VectorIndex.delete looks up the DocumentChunk
1272 # rows by (source_type, source_id, collection_name)
1273 # to learn which FAISS int ids to remove, and
1274 # deletes those matching rows itself. The rows
1275 # must still exist when this runs, or the lookup
1276 # finds nothing and the vectors (and their
1277 # persisted text) are stranded in the FAISS store
1278 # forever. purge_document_chunks below then only
1279 # has to catch whatever purge_document_vectors
1280 # didn't match (e.g. no current FAISS index).
1281 rag.purge_document_vectors(note_id, collection_id)
1282 # purge_document_chunks, not
1283 # remove_document_from_rag: the Document (and
1284 # its DocumentCollection join row) is already
1285 # cascade-deleted, so the join-row lookup in
1286 # remove_document_from_rag would no-op and
1287 # leave the chunks orphaned.
1288 rag.purge_document_chunks(note_id, collection_id)
1289 except Exception:
1290 # exception=True keeps the stack trace. `dbpw` is a
1291 # live local, but frame-local values render only under
1292 # the opt-in LDR_LOGURU_DIAGNOSE dev flag — every
1293 # persistent sink sets diagnose=False — so the password
1294 # cannot leak into logs in normal operation.
1295 logger.opt(exception=True).error(
1296 "Failed to remove RAG entries for note {} "
1297 "from collection {}",
1298 note_id,
1299 collection_id,
1300 )
1302 # Guaranteed backstop: delete any DocumentChunk rows the per-
1303 # collection purge above could not reach (get_rag_service failing
1304 # because the request-thread password is unavailable, a transient
1305 # FAISS/DB error, a stored-config mismatch). chunk_text holds the
1306 # note's content and collection search rehydrates hits purely by
1307 # DocumentChunk.id — a surviving row would keep serving the deleted
1308 # note's text. The matching orphaned vector then resolves to no row
1309 # and is filtered out. Mirrors DocumentDeletionService's backstop.
1310 try:
1311 with get_user_db_session(self.username) as sweep_session:
1312 sweep_session.query(DocumentChunk).filter_by(
1313 source_type="document", source_id=note_id
1314 ).delete(synchronize_session=False)
1315 sweep_session.commit()
1316 except Exception:
1317 logger.opt(exception=True).error(
1318 "Failed to sweep residual chunk rows for note {}", note_id
1319 )
1321 return True
1323 except Exception:
1324 logger.exception("Error deleting note {}", note_id)
1325 raise
1327 def _build_filtered_notes_query(
1328 self,
1329 session: Session,
1330 collection_id: Optional[str],
1331 search: Optional[str],
1332 pinned_only: bool,
1333 ):
1334 """Build the filtered Document query used by both ``list_notes``
1335 and ``count_notes``. Centralising the filter shape keeps the
1336 list response and its ``total`` count in lockstep.
1337 """
1338 source_type_id = self._get_note_source_type_id(session)
1339 query = session.query(Document).filter(
1340 Document.source_type_id == source_type_id
1341 )
1343 if collection_id: 1343 ↛ 1344line 1343 didn't jump to line 1344 because the condition on line 1343 was never true
1344 query = query.join(DocumentCollection).filter(
1345 DocumentCollection.collection_id == collection_id
1346 )
1348 if search:
1349 search_pattern = f"%{self._escape_like(search)}%"
1350 query = query.filter(
1351 or_(
1352 Document.title.ilike(search_pattern, escape="\\"),
1353 Document.text_content.ilike(search_pattern, escape="\\"),
1354 )
1355 )
1357 if pinned_only:
1358 query = query.filter(Document.favorite.is_(True))
1360 return query
1362 def count_notes(
1363 self,
1364 collection_id: Optional[str] = None,
1365 search: Optional[str] = None,
1366 pinned_only: bool = False,
1367 ) -> int:
1368 """Count notes matching the same filter as ``list_notes``.
1370 Used by the list endpoint so the frontend can render real
1371 pagination controls (page-N-of-M, "load more" with bounded
1372 progress, etc.) without having to call ``list_notes`` with a
1373 huge limit just to compute the size.
1374 """
1375 with get_user_db_session(self.username) as session:
1376 return self._build_filtered_notes_query(
1377 session, collection_id, search, pinned_only
1378 ).count()
1380 def list_notes(
1381 self,
1382 collection_id: Optional[str] = None,
1383 search: Optional[str] = None,
1384 pinned_only: bool = False,
1385 limit: int = 100,
1386 offset: int = 0,
1387 ) -> List[Dict[str, Any]]:
1388 """List notes with optional filtering."""
1389 try:
1390 with get_user_db_session(self.username) as session:
1391 query = self._build_filtered_notes_query(
1392 session, collection_id, search, pinned_only
1393 )
1395 query = query.order_by(
1396 Document.favorite.desc(),
1397 Document.updated_at.desc(),
1398 # Stable tiebreaker: without it, notes sharing an
1399 # updated_at (bulk-created) have no guaranteed order, so
1400 # offset pagination ("Load more") could repeat or skip
1401 # rows between pages; mirrors the mention/link panels.
1402 Document.id.desc(),
1403 )
1405 query = query.offset(offset).limit(limit)
1406 documents = query.all()
1408 if not documents:
1409 return []
1411 doc_ids = [d.id for d in documents]
1413 # Batched aggregates: replaces 3 per-row sub-queries inside
1414 # _document_to_note_dict with 3 grouped queries total. Default
1415 # limit=100 went from 300 extra queries/page to 3.
1416 collection_counts = dict(
1417 session.query(
1418 DocumentCollection.document_id,
1419 func.count(DocumentCollection.id),
1420 )
1421 .filter(DocumentCollection.document_id.in_(doc_ids))
1422 .group_by(DocumentCollection.document_id)
1423 .all()
1424 )
1425 research_counts = dict(
1426 session.query(
1427 NoteResearch.document_id,
1428 func.count(NoteResearch.id),
1429 )
1430 .filter(NoteResearch.document_id.in_(doc_ids))
1431 .group_by(NoteResearch.document_id)
1432 .all()
1433 )
1434 indexed_rows = (
1435 session.query(DocumentCollection)
1436 .filter(
1437 DocumentCollection.document_id.in_(doc_ids),
1438 DocumentCollection.indexed.is_(True),
1439 )
1440 .all()
1441 )
1442 indexed_by_doc: Dict[str, DocumentCollection] = {}
1443 for row in indexed_rows: 1443 ↛ 1445line 1443 didn't jump to line 1445 because the loop on line 1443 never started
1444 # Mirror prior .first() semantics: first indexed row wins
1445 indexed_by_doc.setdefault(row.document_id, row)
1447 return [
1448 self._document_to_note_dict(
1449 doc,
1450 session,
1451 _collection_count=collection_counts.get(doc.id, 0),
1452 _research_count=research_counts.get(doc.id, 0),
1453 _indexed_coll=indexed_by_doc.get(doc.id),
1454 include_full_content=False,
1455 )
1456 for doc in documents
1457 ]
1459 except Exception:
1460 logger.exception("Error listing notes")
1461 raise
1463 def add_to_collection(self, note_id: str, collection_id: str) -> bool:
1464 """Add a note to a collection."""
1465 try:
1466 with get_user_db_session(self.username) as session:
1467 # Only operate on notes — every other mutating method guards
1468 # with _is_note, but these collection helpers did not, so a
1469 # non-note Document id could get note-collection rows attached.
1470 doc = self._get_note_in_session(session, note_id)
1471 if doc is None:
1472 return False
1474 # Validate the target collection exists before inserting the
1475 # join row. Otherwise a stale/bogus collection_id trips the FK
1476 # constraint and surfaces as a generic 500 instead of a clean
1477 # 404 (mirrors create_note, which filters to valid ids).
1478 collection = (
1479 session.query(Collection)
1480 .filter_by(id=collection_id)
1481 .first()
1482 )
1483 if collection is None:
1484 raise LookupError( # noqa: TRY301 — except logs and re-raises to the route as a 404
1485 f"Collection {collection_id} not found"
1486 )
1488 existing = (
1489 session.query(DocumentCollection)
1490 .filter_by(document_id=note_id, collection_id=collection_id)
1491 .first()
1492 )
1494 if existing:
1495 return False
1497 doc_coll = DocumentCollection(
1498 document_id=note_id,
1499 collection_id=collection_id,
1500 indexed=False,
1501 )
1502 session.add(doc_coll)
1503 session.commit()
1504 logger.info(
1505 f"Added note {note_id} to collection {collection_id}"
1506 )
1507 return True
1509 except IntegrityError:
1510 # TOCTOU on UNIQUE(document_id, collection_id): a concurrent add of
1511 # the same note+collection raced past the existing-row check above.
1512 # get_user_db_session already rolled the shared session back; return
1513 # the idempotent "already a member" outcome (route → 409) instead of
1514 # letting the constraint violation surface as a 500.
1515 return False
1516 except Exception:
1517 logger.exception("Error adding note {} to collection", note_id)
1518 raise
1520 def remove_from_collection(self, note_id: str, collection_id: str) -> bool:
1521 """Remove a note from a collection.
1523 Raises ValueError when ``collection_id`` is the system Notes
1524 collection: it is the persistent home of every note (see
1525 ``_get_or_create_notes_collection``), and the deletion services
1526 rely on that invariant — a note unlinked from Notes but still in
1527 a user collection would be hard-deleted as an "orphan" when that
1528 collection is deleted.
1529 """
1530 try:
1531 was_indexed = False
1532 with get_user_db_session(self.username) as session:
1533 doc = self._get_note_in_session(session, note_id)
1534 if doc is None:
1535 return False
1537 collection = (
1538 session.query(Collection)
1539 .filter_by(id=collection_id)
1540 .first()
1541 )
1542 if (
1543 collection is not None
1544 and collection.collection_type == "notes"
1545 ):
1546 raise ValueError( # noqa: TRY301 — needs the db session; except logs and re-raises
1547 "Cannot remove a note from the Notes collection — "
1548 "it is the permanent home of every note"
1549 )
1551 doc_coll = (
1552 session.query(DocumentCollection)
1553 .filter_by(document_id=note_id, collection_id=collection_id)
1554 .first()
1555 )
1557 link_existed = doc_coll is not None
1558 if doc_coll:
1559 # Purge if this collection is flagged indexed OR the note
1560 # still has chunk rows here — the chunk-row half catches the
1561 # edit->reindex window where indexed=False but the pre-edit
1562 # chunks/vectors still exist. Captured before the link is
1563 # deleted.
1564 was_indexed = bool(doc_coll.indexed) or (
1565 session.query(DocumentChunk.id)
1566 .filter_by(
1567 source_type="document",
1568 source_id=note_id,
1569 collection_name=f"collection_{collection_id}",
1570 )
1571 .first()
1572 is not None
1573 )
1574 session.delete(doc_coll)
1575 session.commit()
1576 logger.info(
1577 f"Removed note {note_id} from collection {collection_id}"
1578 )
1579 else:
1580 # The link is already gone (a concurrent removal). Do NOT
1581 # return early: a racing index_document could have (re)created
1582 # chunk rows for this (note, collection) AFTER the link was
1583 # deleted, and there is no other route-level retry for that
1584 # pair — an early return would leave the removed note's
1585 # plaintext searchable here forever. Fall through to the
1586 # unconditional backstop chunk sweep below (which removes the
1587 # plaintext rows); the full RAG vector purge is skipped since
1588 # there is no link to unlink.
1589 was_indexed = False
1591 # Best-effort FAISS + chunk cleanup for the collection the note was
1592 # unlinked from — mirrors delete_note. Without it, the note's chunk
1593 # rows and vectors linger in that collection's index, so the removed
1594 # note stays fully searchable there. Vectors BEFORE chunk rows (the
1595 # int-id lookup needs the rows to still exist). Uses get_rag_service
1596 # so THAT collection's stored embedding model resolves the right
1597 # index hash. Failures are logged, not surfaced (the unlink already
1598 # committed).
1599 if was_indexed:
1600 dbpw = _capture_request_db_password(self.username)
1601 from ...services.rag_service_factory import get_rag_service
1603 try:
1604 with get_rag_service(
1605 self.username,
1606 collection_id=collection_id,
1607 db_password=dbpw,
1608 ) as rag:
1609 rag.purge_document_vectors(note_id, collection_id)
1610 rag.purge_document_chunks(note_id, collection_id)
1611 except Exception:
1612 # exception=True keeps the stack trace. `dbpw` (the DB
1613 # password) is a live local, but frame-local values render
1614 # only under the opt-in LDR_LOGURU_DIAGNOSE dev flag — every
1615 # persistent sink sets diagnose=False — so it cannot leak.
1616 logger.opt(exception=True).error(
1617 "Failed to purge RAG entries for note {} removed from "
1618 "collection {}",
1619 note_id,
1620 collection_id,
1621 )
1623 # Guaranteed backstop, scoped to THIS collection only (the note
1624 # survives in its other collections): delete any DocumentChunk rows
1625 # the purge above could not reach, so the unlinked note's text stops
1626 # surfacing in this collection's search even when the RAG purge
1627 # failed. Mirrors DocumentDeletionService's scoped backstop.
1628 try:
1629 with get_user_db_session(self.username) as sweep_session:
1630 sweep_session.query(DocumentChunk).filter_by(
1631 source_type="document",
1632 source_id=note_id,
1633 collection_name=f"collection_{collection_id}",
1634 ).delete(synchronize_session=False)
1635 sweep_session.commit()
1636 except Exception:
1637 logger.opt(exception=True).error(
1638 "Failed to sweep residual chunk rows for note {} in "
1639 "collection {}",
1640 note_id,
1641 collection_id,
1642 )
1643 # False when the link was already gone (nothing to unlink), True when
1644 # this call performed the removal — preserves the prior contract.
1645 return link_existed
1647 except Exception:
1648 logger.exception("Error removing note {} from collection", note_id)
1649 raise
1651 def get_note_collections(self, note_id: str) -> List[Dict[str, Any]]:
1652 """Get collections a note belongs to."""
1653 try:
1654 with get_user_db_session(self.username) as session:
1655 # Use JOIN to avoid N+1 query issue
1656 results = (
1657 session.query(Collection, DocumentCollection)
1658 .join(
1659 DocumentCollection,
1660 Collection.id == DocumentCollection.collection_id,
1661 )
1662 .filter(DocumentCollection.document_id == note_id)
1663 .all()
1664 )
1666 return [
1667 {
1668 "id": coll.id,
1669 "name": coll.name,
1670 # The UI hides the remove control for the system
1671 # Notes collection (every note's persistent home;
1672 # remove_from_collection refuses it server-side).
1673 "collection_type": coll.collection_type,
1674 "indexed": dc.indexed,
1675 "chunk_count": dc.chunk_count or 0,
1676 }
1677 for coll, dc in results
1678 ]
1680 except Exception:
1681 logger.exception("Error getting collections for note {}", note_id)
1682 raise
1684 def get_note_research(self, note_id: str) -> List[Dict[str, Any]]:
1685 """Get research runs triggered from a note."""
1686 try:
1687 with get_user_db_session(self.username) as session:
1688 note_research = (
1689 session.query(NoteResearch)
1690 .options(joinedload(NoteResearch.research))
1691 .filter_by(document_id=note_id)
1692 .order_by(NoteResearch.display_order.asc())
1693 .all()
1694 )
1696 return [
1697 {
1698 "id": nr.id,
1699 "research_id": nr.research_id,
1700 "search_engine": (nr.research.research_meta or {})
1701 .get("submission", {})
1702 .get("search_engine")
1703 if nr.research
1704 else None,
1705 "research_mode": nr.research.mode
1706 if nr.research
1707 else None,
1708 "query_used": nr.research.query
1709 if nr.research
1710 else None,
1711 "display_order": nr.display_order,
1712 "is_collapsed": nr.is_collapsed,
1713 "created_at": nr.created_at.isoformat()
1714 if nr.created_at
1715 else None,
1716 }
1717 for nr in note_research
1718 ]
1720 except Exception:
1721 logger.exception("Error getting research for note {}", note_id)
1722 raise
1724 # Max retries on display_order UNIQUE collision. The old value of 5
1725 # was contention-fragile: colliding threads re-read MAX(display_order)
1726 # in lockstep, so under 6+ concurrent same-note links one thread could
1727 # lose all 5 races and 500 the user (dropping the link). Each retry can
1728 # only fail because another writer landed at the same display_order, and
1729 # the number of concurrent NoteResearch inserts is itself bounded by
1730 # MAX_RESEARCH_PER_NOTE — so a budget comfortably above that, combined
1731 # with the randomized jitter below, makes losing every race effectively
1732 # impossible while never masking a non-retryable error.
1733 _LINK_RESEARCH_MAX_RETRIES = MAX_RESEARCH_PER_NOTE + 10
1735 def link_research_to_note(
1736 self,
1737 note_id: str,
1738 research_id: str,
1739 ) -> int:
1740 """Link a research run to a note.
1742 Concurrency: two same-user requests (e.g. double-click or two
1743 tabs) can both observe the same MAX(display_order) and try to
1744 insert with the same display_order. The UNIQUE constraint on
1745 (document_id, display_order) added in migration 0021 turns the
1746 race into an IntegrityError; we catch it, rollback, recompute
1747 MAX, and retry up to a small bound. The
1748 (document_id, research_id) UniqueConstraint is separate — that
1749 one signals "research already linked", which surfaces as
1750 IntegrityError too. We distinguish the two by looking for the
1751 ``display_order`` token in the error: SQLite reports the offending
1752 COLUMNS (not the constraint name) and Postgres reports the constraint
1753 name ``uix_note_research_display_order`` — both contain
1754 ``display_order``, while the duplicate-link and FK errors do not. Only
1755 the display_order collision is retryable; everything else surfaces
1756 immediately so a legitimate duplicate link (or an FK violation) isn't
1757 masked by burning retries.
1758 """
1759 for attempt in range(self._LINK_RESEARCH_MAX_RETRIES): 1759 ↛ 1860line 1759 didn't jump to line 1860 because the loop on line 1759 didn't complete
1760 session = None
1761 try:
1762 with get_user_db_session(self.username) as session:
1763 # Every other mutating method guards with _is_note;
1764 # without this, a NoteResearch row can be attached to
1765 # any Document (e.g. an uploaded PDF) — orphaned rows
1766 # the notes UI never renders.
1767 doc = self._get_note_in_session(session, note_id)
1768 if doc is None:
1769 raise ValueError( # noqa: TRY301 — needs the db session; except rolls back and re-raises
1770 f"Note {note_id} not found"
1771 )
1773 # Enforce MAX_RESEARCH_PER_NOTE where rows are created
1774 # (this is the only NoteResearch creation site). The
1775 # cap previously existed only in reorder_note_research,
1776 # so a note could grow past it and then every reorder
1777 # of the (necessarily full) list failed on the same
1778 # cap — permanently un-reorderable.
1779 existing_count = (
1780 session.query(func.count(NoteResearch.id))
1781 .filter_by(document_id=note_id)
1782 .scalar()
1783 )
1784 if existing_count >= MAX_RESEARCH_PER_NOTE:
1785 raise ValueError( # noqa: TRY301 — needs the db session; except rolls back and re-raises
1786 f"too many linked researches "
1787 f"(max {MAX_RESEARCH_PER_NOTE})"
1788 )
1790 # Use MAX(display_order) + 1, not COUNT, so deleting an
1791 # earlier link can't make the next insert reuse an
1792 # existing display_order. Research runs CASCADE-delete
1793 # NoteResearch rows when a ResearchHistory is removed,
1794 # so COUNT < MAX+1 is a real possibility.
1795 max_order = (
1796 session.query(
1797 func.coalesce(
1798 func.max(NoteResearch.display_order), -1
1799 )
1800 )
1801 .filter_by(document_id=note_id)
1802 .scalar()
1803 )
1805 note_research = NoteResearch(
1806 document_id=note_id,
1807 research_id=research_id,
1808 display_order=max_order + 1,
1809 )
1810 session.add(note_research)
1811 session.commit()
1812 logger.info(
1813 "Linked research {} to note {}", research_id, note_id
1814 )
1815 return note_research.id
1817 except IntegrityError as exc:
1818 # The shared per-user request session is NOT rolled back by the
1819 # context manager on exception (it only closes a session it
1820 # created), so without an explicit rollback the next retry reuses
1821 # a poisoned session and fails identically.
1822 self._rollback_quietly(session)
1823 msg = str(exc.orig).lower() if exc.orig else str(exc).lower()
1824 # Only a (document_id, display_order) collision is retryable.
1825 # The duplicate (document_id, research_id) link and FK violations
1826 # contain no "display_order" token → surface immediately.
1827 if "display_order" not in msg:
1828 raise
1829 if attempt + 1 == self._LINK_RESEARCH_MAX_RETRIES:
1830 logger.exception(
1831 "Giving up linking research {} to note {} after "
1832 "{} display_order collisions",
1833 research_id,
1834 note_id,
1835 self._LINK_RESEARCH_MAX_RETRIES,
1836 )
1837 raise
1838 logger.debug(
1839 "display_order collision linking research {} to note "
1840 "{} (attempt {}); retrying",
1841 research_id,
1842 note_id,
1843 attempt + 1,
1844 )
1845 # Backoff with randomized jitter so colliding threads
1846 # don't re-read MAX(display_order) in perfect lockstep and
1847 # collide again on the very next attempt. Spreading the
1848 # retries breaks the synchronized contention that made a
1849 # fixed, no-backoff budget exhaustible under load.
1850 # noqa S311: jitter for retry-backoff scheduling, not a
1851 # security/crypto context — non-CSPRNG randomness is fine.
1852 time.sleep(random.uniform(0, 0.01 * (attempt + 1))) # noqa: S311
1853 continue
1854 except Exception:
1855 self._rollback_quietly(session)
1856 logger.exception("Error linking research to note {}", note_id)
1857 raise
1859 # Unreachable — the loop either returns or raises.
1860 raise RuntimeError("link_research_to_note exhausted retries")
1862 def get_notes_for_research(
1863 self, research_id: str
1864 ) -> Optional[List[Dict[str, Any]]]:
1865 """List the notes linked to a research run, newest link first.
1867 The reverse direction of ``get_research_for_note`` — powers the
1868 Notes panel on the research results page. Returns ``None`` when
1869 the research run doesn't exist (the route maps that to 404,
1870 distinct from "exists but has no notes yet" → ``[]``).
1872 Column-projected like ``get_backlinks``: ``text_content`` can be
1873 megabytes per note, and this renders a card list that needs only
1874 a short preview.
1875 """
1876 from ....database.models import ResearchHistory
1878 preview_len = self.LIST_CONTENT_PREVIEW_CHARS
1879 with get_user_db_session(self.username) as session:
1880 if (
1881 session.query(ResearchHistory.id)
1882 .filter_by(id=research_id)
1883 .first()
1884 is None
1885 ):
1886 return None
1887 rows = (
1888 session.query(
1889 Document.id,
1890 Document.title,
1891 func.substr(Document.text_content, 1, preview_len + 1),
1892 Document.updated_at,
1893 NoteResearch.created_at,
1894 )
1895 .join(NoteResearch, NoteResearch.document_id == Document.id)
1896 .filter(NoteResearch.research_id == research_id)
1897 .order_by(NoteResearch.created_at.desc())
1898 .all()
1899 )
1900 notes = []
1901 for note_id, title, preview, updated_at, linked_at in rows:
1902 preview = self._truncate_preview(preview, preview_len)
1903 notes.append(
1904 {
1905 "id": note_id,
1906 "title": title,
1907 "content_preview": preview,
1908 "updated_at": updated_at.isoformat()
1909 if updated_at
1910 else None,
1911 "linked_at": linked_at.isoformat()
1912 if linked_at
1913 else None,
1914 }
1915 )
1916 return notes
1918 def _create_note_with_cleanup(
1919 self, title, content, tags, link_fn, context_label
1920 ):
1921 """Create a note and run its post-create ``link_fn`` step
1922 atomically: if linking raises, delete the just-created note so no
1923 orphan survives, then re-raise the original error.
1925 Shared by ``create_note_for_research`` /
1926 ``create_note_for_document`` — the two paths that create-then-link,
1927 where a mid-step failure would otherwise leave a note the user
1928 never asked for. ``context_label`` is the full link-step phrase up
1929 to (and including) its preposition, so the log reads naturally
1930 (e.g. ``"Linking research <id> to"`` → "... to fresh note ...").
1931 """
1932 note_id = self.create_note(title=title, content=content, tags=tags)
1933 try:
1934 link_fn(note_id)
1935 except Exception:
1936 logger.exception(
1937 "{} fresh note {} failed; removing the note so the "
1938 "operation stays all-or-nothing",
1939 context_label,
1940 note_id,
1941 )
1942 try:
1943 self.delete_note(note_id)
1944 except Exception:
1945 # Both halves failed — surface the original linking error;
1946 # the orphan note is visible (not silently lost data).
1947 logger.exception(
1948 "Cleanup of orphan note {} also failed", note_id
1949 )
1950 raise
1951 return note_id
1953 def create_note_for_research(
1954 self,
1955 research_id: str,
1956 title: Optional[str] = None,
1957 content: Optional[str] = None,
1958 tags: Optional[List[str]] = None,
1959 quote: Optional[str] = None,
1960 prefix: Optional[str] = None,
1961 suffix: Optional[str] = None,
1962 ) -> str:
1963 """Create a note and link it to a research run, all-or-nothing.
1965 Used by the results page ("Add note", "Save as note", clip). The
1966 research row is checked FIRST so a bogus id gets a clean
1967 ``LookupError`` (route → 404) instead of creating an orphan note
1968 and then FK-failing on the link. If the link step fails anyway
1969 (e.g. a concurrent research delete), the just-created note is
1970 removed so the caller never observes a half-done operation.
1972 ``title``/``content`` default to a starter derived from the
1973 research query, so the "Add note" button needs no user input
1974 before navigating into the editor.
1975 """
1976 from ....database.models import ResearchHistory
1978 with get_user_db_session(self.username) as session:
1979 research = (
1980 session.query(ResearchHistory.id, ResearchHistory.query)
1981 .filter_by(id=research_id)
1982 .first()
1983 )
1984 if research is None:
1985 raise LookupError("Research not found")
1987 query_text = (research.query or "").strip()
1988 if not title:
1989 title = f"Notes on: {query_text}"[:MAX_TITLE_LENGTH]
1990 if not content:
1991 label = escape_markdown_link_label(query_text or research_id)
1992 content = (
1993 f"> Notes on the research run "
1994 f"[{label}](/results/{research_id}).\n\n"
1995 )
1997 def _link(new_note_id):
1998 self.link_research_to_note(new_note_id, research_id)
1999 if quote:
2000 # An inline annotation: also record the quote anchor so the
2001 # results page can highlight the passage (NULL-quote
2002 # references are plain links and skip this).
2003 self._add_reference(
2004 new_note_id,
2005 target_research_id=research_id,
2006 quote=quote,
2007 prefix=prefix,
2008 suffix=suffix,
2009 )
2011 return self._create_note_with_cleanup(
2012 title, content, tags, _link, f"Linking research {research_id} to"
2013 )
2015 def _add_reference(
2016 self,
2017 note_id: str,
2018 target_document_id: Optional[str] = None,
2019 target_research_id: Optional[str] = None,
2020 quote: Optional[str] = None,
2021 prefix: Optional[str] = None,
2022 suffix: Optional[str] = None,
2023 ) -> int:
2024 """Insert a NoteReference row (see the model for semantics)."""
2025 with get_user_db_session(self.username) as session:
2026 ref = NoteReference(
2027 note_id=note_id,
2028 target_document_id=target_document_id,
2029 target_research_id=target_research_id,
2030 quote=quote,
2031 prefix=prefix,
2032 suffix=suffix,
2033 )
2034 session.add(ref)
2035 try:
2036 session.commit()
2037 except Exception:
2038 self._rollback_quietly(session)
2039 raise
2040 return ref.id
2042 def get_notes_for_document(
2043 self, document_id: str
2044 ) -> Optional[List[Dict[str, Any]]]:
2045 """List the notes referencing a library document, newest first.
2047 Returns ``None`` when the document doesn't exist (route → 404).
2048 Column-projected like ``get_notes_for_research``; a note with
2049 several references (e.g. two annotations) appears once.
2050 """
2051 preview_len = self.LIST_CONTENT_PREVIEW_CHARS
2052 with get_user_db_session(self.username) as session:
2053 if (
2054 session.query(Document.id).filter_by(id=document_id).first()
2055 is None
2056 ):
2057 return None
2058 rows = (
2059 session.query(
2060 Document.id,
2061 Document.title,
2062 func.substr(Document.text_content, 1, preview_len + 1),
2063 Document.updated_at,
2064 func.max(NoteReference.created_at),
2065 )
2066 .join(NoteReference, NoteReference.note_id == Document.id)
2067 .filter(NoteReference.target_document_id == document_id)
2068 .group_by(Document.id)
2069 .order_by(func.max(NoteReference.created_at).desc())
2070 .all()
2071 )
2072 notes = []
2073 for note_id, title, preview, updated_at, linked_at in rows:
2074 preview = self._truncate_preview(preview, preview_len)
2075 notes.append(
2076 {
2077 "id": note_id,
2078 "title": title,
2079 "content_preview": preview,
2080 "updated_at": updated_at.isoformat()
2081 if updated_at
2082 else None,
2083 "linked_at": linked_at.isoformat()
2084 if linked_at
2085 else None,
2086 }
2087 )
2088 return notes
2090 def create_note_for_document(
2091 self,
2092 document_id: str,
2093 title: Optional[str] = None,
2094 content: Optional[str] = None,
2095 tags: Optional[List[str]] = None,
2096 quote: Optional[str] = None,
2097 prefix: Optional[str] = None,
2098 suffix: Optional[str] = None,
2099 ) -> str:
2100 """Create a note referencing a library document, all-or-nothing.
2102 The document-side twin of ``create_note_for_research``. Targets
2103 must be non-note documents: annotating a NOTE is deliberately
2104 unsupported — notes are mutable, so quote anchors would drift on
2105 every edit (the re-anchoring problem); library documents' extracted
2106 text is immutable, like research reports.
2107 """
2108 with get_user_db_session(self.username) as session:
2109 row = (
2110 session.query(Document.id, Document.title)
2111 .filter_by(id=document_id)
2112 .first()
2113 )
2114 if row is None:
2115 raise LookupError("Document not found")
2116 note_source_type_id = self._get_note_source_type_id(session)
2117 is_note = (
2118 session.query(Document.id)
2119 .filter_by(id=document_id, source_type_id=note_source_type_id)
2120 .first()
2121 is not None
2122 )
2123 if is_note:
2124 raise ValueError(
2125 "notes cannot be annotated — their content is mutable, "
2126 "so quote anchors would drift on edit; annotate library "
2127 "documents or research reports instead"
2128 )
2129 doc_title = (row.title or "").strip() or document_id
2131 if not title:
2132 title = f"Notes on: {doc_title}"[:MAX_TITLE_LENGTH]
2133 if not content:
2134 label = escape_markdown_link_label(doc_title)
2135 content = (
2136 f"> Notes on the document "
2137 f"[{label}](/library/document/{document_id}).\n\n"
2138 )
2140 def _link(new_note_id):
2141 self._add_reference(
2142 new_note_id,
2143 target_document_id=document_id,
2144 quote=quote,
2145 prefix=prefix,
2146 suffix=suffix,
2147 )
2149 return self._create_note_with_cleanup(
2150 title,
2151 content,
2152 tags,
2153 _link,
2154 f"Referencing document {document_id} from",
2155 )
2157 def get_annotations_for_target(
2158 self,
2159 document_id: Optional[str] = None,
2160 research_id: Optional[str] = None,
2161 ) -> List[Dict[str, Any]]:
2162 """List the anchored references (inline annotations) for a target.
2164 Existence of the target is the caller's concern (the routes check
2165 it for the 404); a valid target with no anchors returns ``[]``.
2166 """
2167 preview_len = self.LIST_CONTENT_PREVIEW_CHARS
2168 with get_user_db_session(self.username) as session:
2169 q = (
2170 session.query(
2171 NoteReference.note_id,
2172 NoteReference.quote,
2173 NoteReference.prefix,
2174 NoteReference.suffix,
2175 NoteReference.created_at,
2176 Document.title,
2177 func.substr(Document.text_content, 1, preview_len + 1),
2178 )
2179 .join(Document, Document.id == NoteReference.note_id)
2180 .filter(NoteReference.quote.isnot(None))
2181 )
2182 if document_id is not None:
2183 q = q.filter(NoteReference.target_document_id == document_id)
2184 else:
2185 q = q.filter(NoteReference.target_research_id == research_id)
2186 annotations = []
2187 for row in q.order_by(NoteReference.created_at.asc()).all():
2188 (
2189 note_id,
2190 quote,
2191 prefix,
2192 suffix,
2193 created_at,
2194 title,
2195 preview,
2196 ) = row
2197 preview = self._truncate_preview(preview, preview_len)
2198 annotations.append(
2199 {
2200 "note_id": note_id,
2201 "quote": quote or "",
2202 "prefix": prefix or "",
2203 "suffix": suffix or "",
2204 "created_at": created_at.isoformat()
2205 if created_at
2206 else None,
2207 "note_title": title,
2208 "comment_preview": preview,
2209 }
2210 )
2211 return annotations
2213 def has_annotation(
2214 self,
2215 note_id: str,
2216 document_id: Optional[str] = None,
2217 research_id: Optional[str] = None,
2218 ) -> bool:
2219 """True when an anchored reference ties note_id to the target
2220 (the DELETE routes' precondition)."""
2221 with get_user_db_session(self.username) as session:
2222 q = session.query(NoteReference.id).filter(
2223 NoteReference.note_id == note_id,
2224 NoteReference.quote.isnot(None),
2225 )
2226 if document_id is not None:
2227 q = q.filter(NoteReference.target_document_id == document_id)
2228 else:
2229 q = q.filter(NoteReference.target_research_id == research_id)
2230 return q.first() is not None
2232 def update_note_research(
2233 self,
2234 note_id: str,
2235 research_id: str,
2236 is_collapsed: Optional[bool] = None,
2237 ) -> bool:
2238 """Update a NoteResearch row. Currently supports toggling is_collapsed.
2240 Returns True if the row was found and updated, False otherwise.
2241 """
2242 try:
2243 with get_user_db_session(self.username) as session:
2244 nr = (
2245 session.query(NoteResearch)
2246 .filter_by(document_id=note_id, research_id=research_id)
2247 .first()
2248 )
2249 if not nr:
2250 return False
2251 if is_collapsed is not None:
2252 nr.is_collapsed = bool(is_collapsed)
2253 session.commit()
2254 return True
2255 except Exception:
2256 logger.exception(
2257 f"Error updating note_research for {note_id}/{research_id}"
2258 )
2259 raise
2261 def reorder_note_research(
2262 self, note_id: str, research_ids: List[str]
2263 ) -> bool:
2264 """Atomically reorder NoteResearch rows for a note so their
2265 display_order matches the position in research_ids.
2267 Validates every id belongs to the note before updating.
2268 Returns True on success.
2270 Two-pass write because of the
2271 ``UniqueConstraint('document_id', 'display_order')`` added in
2272 migration 0021: assigning the final order directly would hit
2273 an intermediate state where two rows share the same
2274 display_order (e.g. swapping idx 0 and 2 would try to put
2275 idx-2 at 0 while idx-0 is still at 0). The first pass moves
2276 every row into the negative display_order range — guaranteed
2277 not to collide with the positive target values — and the
2278 second pass assigns the final positive values.
2279 """
2280 if not isinstance(research_ids, list): 2280 ↛ 2281line 2280 didn't jump to line 2281 because the condition on line 2280 was never true
2281 raise ValueError("research_ids must be a list")
2282 if len(research_ids) > MAX_RESEARCH_PER_NOTE: 2282 ↛ 2283line 2282 didn't jump to line 2283 because the condition on line 2282 was never true
2283 raise ValueError(
2284 f"too many research_ids (max {MAX_RESEARCH_PER_NOTE})"
2285 )
2286 # A duplicate id collapses in a set, so a set-equality check alone would
2287 # accept e.g. ['A','B','A'] for a note linking {A,B} and then assign A
2288 # two positions — silently corrupting the order. Reject duplicates.
2289 if len(research_ids) != len(set(research_ids)):
2290 return False
2291 session = None
2292 try:
2293 with get_user_db_session(self.username) as session:
2294 existing = (
2295 session.query(NoteResearch)
2296 .filter_by(document_id=note_id)
2297 .all()
2298 )
2299 existing_ids = {nr.research_id for nr in existing}
2300 if set(research_ids) != existing_ids:
2301 return False
2303 by_id = {nr.research_id: nr for nr in existing}
2304 # Pass 1: park every row in the negative range so the
2305 # second pass can assign 0..N-1 without colliding with
2306 # the existing positive display_orders.
2307 for idx, rid in enumerate(research_ids):
2308 by_id[rid].display_order = -(idx + 1)
2309 session.flush()
2310 # Pass 2: final positive values.
2311 for idx, rid in enumerate(research_ids):
2312 by_id[rid].display_order = idx
2313 session.commit()
2314 return True
2315 except Exception:
2316 # Roll back so a failed flush/commit doesn't leave the shared
2317 # request session poisoned for the rest of the request.
2318 self._rollback_quietly(session)
2319 logger.exception("Error reordering research for note {}", note_id)
2320 raise
2322 # Sentinel used by _document_to_note_dict to distinguish "caller passed
2323 # None as the precomputed value (i.e. document is not indexed)" from
2324 # "caller did not pass anything; query the DB". Plain None can't carry
2325 # both meanings.
2326 _UNSET = object()
2328 # Upper bound on body size returned in list-view responses. Frontend
2329 # only ever shows the first ~150 chars; shipping the entire ``text_content``
2330 # column meant a 200-note page could download multiple MB. The 300-char
2331 # cap leaves headroom over the UI cap so the truncation isn't visible
2332 # at the cut-off boundary.
2333 LIST_CONTENT_PREVIEW_CHARS = 300
2335 def _document_to_note_dict(
2336 self,
2337 document: Document,
2338 session: Session,
2339 _collection_count: Any = _UNSET,
2340 _research_count: Any = _UNSET,
2341 _indexed_coll: Any = _UNSET,
2342 include_full_content: bool = True,
2343 ) -> Dict[str, Any]:
2344 """Convert a Document to a note dict.
2346 The three ``_``-prefixed kwargs let list_notes pre-batch the
2347 aggregate queries (count of collections / research links / first
2348 indexed-collection row) into 3 grouped SQL statements instead of
2349 3*N per-row sub-queries. Single-doc callers (get_note) leave them
2350 unset and pay the per-row cost.
2351 """
2352 if _collection_count is self._UNSET:
2353 _collection_count = (
2354 session.query(DocumentCollection)
2355 .filter_by(document_id=document.id)
2356 .count()
2357 )
2358 if _research_count is self._UNSET:
2359 _research_count = (
2360 session.query(NoteResearch)
2361 .filter_by(document_id=document.id)
2362 .count()
2363 )
2364 if _indexed_coll is self._UNSET:
2365 _indexed_coll = (
2366 session.query(DocumentCollection)
2367 .filter_by(document_id=document.id, indexed=True)
2368 .first()
2369 )
2371 collection_count = _collection_count
2372 research_count = _research_count
2373 indexed_coll = _indexed_coll
2375 text_content = document.text_content or ""
2376 if include_full_content:
2377 content_value: Optional[str] = text_content
2378 content_preview: Optional[str] = None
2379 else:
2380 content_value = None
2381 content_preview = self._truncate_preview(
2382 text_content, self.LIST_CONTENT_PREVIEW_CHARS
2383 )
2385 return {
2386 "id": document.id,
2387 "title": document.title,
2388 "content": content_value,
2389 "content_preview": content_preview,
2390 "tags": document.tags or [],
2391 "pinned": document.favorite,
2392 "is_indexed": indexed_coll is not None,
2393 "chunk_count": indexed_coll.chunk_count if indexed_coll else 0,
2394 "last_indexed_at": indexed_coll.last_indexed_at.isoformat()
2395 if indexed_coll and indexed_coll.last_indexed_at
2396 else None,
2397 "created_at": document.created_at.isoformat()
2398 if document.created_at
2399 else None,
2400 "updated_at": document.updated_at.isoformat()
2401 if document.updated_at
2402 else None,
2403 "collection_count": collection_count,
2404 "research_count": research_count,
2405 }
2407 # =========================================================================
2408 # Version History Methods
2409 # =========================================================================
2411 def _create_version_snapshot_in_session(
2412 self,
2413 session: Session,
2414 note_id: str,
2415 title: str,
2416 content: str,
2417 tags: Optional[List[str]],
2418 change_type: str,
2419 change_summary: Optional[str] = None,
2420 ) -> "tuple[str, bool]":
2421 """Create a NoteVersion row in an existing session.
2423 Caller is responsible for committing. Used by ``restore_with_bookends``
2424 to keep PRE_RESTORE + content update + RESTORE in a single
2425 transaction so process death between writes can't lose the audit
2426 trail. Dedup is preserved via the (document_id, content_hash)
2427 check; on a hit, returns the existing version id without inserting.
2429 Returns ``(version_id, created)``: ``created`` is False when the
2430 dedup check absorbed the snapshot into an existing row. Callers
2431 that schedule the async change-summary worker MUST check it —
2432 passing a dedup-absorbed (historical) version id to the worker
2433 would overwrite that old row's change_summary with a description
2434 of a much later edit.
2436 Note that ``content_hash`` is computed over title + content + tags
2437 via ``_compute_version_hash`` so title-only or tag-only edits
2438 produce a distinct hash and aren't silently deduped against an
2439 existing snapshot. The column name stays ``content_hash`` for
2440 schema compatibility but its semantics are "version-state hash".
2441 """
2442 # NoteVersion.content/tags are NOT NULL (note.py), but the source
2443 # this snapshots — Document.text_content — IS nullable, and the
2444 # restore bookend path passes document fields through uncoerced.
2445 # Coerce here so the version table is never stricter than the row
2446 # it captures (a NULL would raise IntegrityError on flush). Title
2447 # is already NOT NULL upstream.
2448 content = content or ""
2449 tags = tags or []
2450 # PRE_RESTORE and RESTORE are audit bookends: by definition their
2451 # state hash matches an existing version row (PRE_RESTORE = current
2452 # state, which IS the prior MANUAL_SAVE; RESTORE = the target
2453 # version's state). Dedup + UNIQUE(document_id, content_hash) would
2454 # silently drop the bookend, erasing the "user restored here" audit
2455 # signal. Salt the hash with a fresh UUID for these change types so
2456 # each row is guaranteed to insert.
2457 #
2458 # INVARIANT WARNING for future contributors: for PRE_RESTORE and
2459 # RESTORE rows, ``content_hash`` is NOT reproducible from
2460 # ``(title, content, tags)`` via _compute_version_hash — the salt
2461 # is hash-only and is NOT persisted into the ``tags`` column
2462 # (clean user-facing data). Any integrity checker that walks
2463 # NoteVersion rows and recomputes hashes MUST filter on
2464 # ``change_type NOT IN ('pre_restore', 'restore')`` before
2465 # comparing — otherwise it will see every restore event as
2466 # corrupted. Removing the salt to "fix" the divergence would
2467 # silently reintroduce the audit-row-drop bug this guards
2468 # against; if you find yourself reaching for that, add a
2469 # change_type column to the UNIQUE constraint instead.
2470 is_audit_bookend = change_type in _AUDIT_BOOKEND_TYPES
2471 if is_audit_bookend:
2472 salted_tags = list(tags or []) + [
2473 f"__{change_type}__{uuid.uuid4()}"
2474 ]
2475 content_hash = self._compute_version_hash(
2476 title, content, salted_tags
2477 )
2478 else:
2479 content_hash = self._compute_version_hash(title, content, tags)
2480 existing = (
2481 session.query(NoteVersion)
2482 .filter_by(document_id=note_id, content_hash=content_hash)
2483 .first()
2484 )
2485 if existing:
2486 return (existing.id, False)
2488 version_id = str(uuid.uuid4())
2489 version = NoteVersion(
2490 id=version_id,
2491 document_id=note_id,
2492 title=title,
2493 content=content,
2494 tags=tags or [],
2495 change_type=change_type,
2496 change_summary=change_summary,
2497 content_hash=content_hash,
2498 )
2499 session.add(version)
2500 session.flush()
2501 return (version_id, True)
2503 def _mark_note_stale_for_reindex_in_session(
2504 self, session: Session, note_id: str
2505 ) -> None:
2506 """Mark a note's embeddings stale after a content/title change.
2508 Two indexed-state sources must move together:
2510 * ``DocumentCollection.indexed`` — the legacy flag the auto-index
2511 worker scans (it runs with ``force_reindex=False``, so a stale
2512 ``indexed=True`` makes it short-circuit and keep serving the
2513 pre-edit embedding).
2514 * ``RagDocumentStatus`` — row-existence is the *canonical* "indexed"
2515 marker that the RAG status route and ``get_rag_stats`` read.
2517 ``index_document`` writes BOTH on index; the edit path must clear
2518 BOTH or the status report shows an edited note as still-indexed
2519 until (and unless) a re-index actually runs. Deleting the status
2520 row matches the model's "no row = not indexed" contract — an edited
2521 note genuinely isn't currently indexed; the worker re-creates the
2522 row when it re-embeds. Caller commits.
2523 """
2524 session.query(DocumentCollection).filter_by(document_id=note_id).update(
2525 {DocumentCollection.indexed: False},
2526 synchronize_session=False,
2527 )
2528 session.query(RagDocumentStatus).filter_by(document_id=note_id).delete(
2529 synchronize_session=False
2530 )
2532 def _prune_versions_in_session(
2533 self, session: Session, note_id: str
2534 ) -> "tuple[int, int]":
2535 """Prune ordinary and restore-bookend versions to independent caps.
2537 Caller is responsible for committing. Used by both ``update_note``
2538 (after its in-session snapshot write) and ``restore_with_bookends``
2539 (which adds PRE_RESTORE + RESTORE rows without consuming the ordinary
2540 version budget).
2542 Returns ``(ordinary_pruned, bookend_pruned)`` — the counts of rows
2543 deleted in *this* call. The deletes are only flushed here, not
2544 committed: if the caller's transaction later rolls back, these
2545 rows survive. Callers must NOT log these counts themselves until
2546 after their own ``session.commit()`` succeeds — logging here (at
2547 flush time) would claim a prune that a subsequent rollback undoes,
2548 turning the one operator-facing signal for version loss into a
2549 false positive on exactly the failure paths it exists to surface.
2550 See ``update_note`` and ``restore_with_bookends`` for the
2551 post-commit logging call.
2552 """
2553 ordinary_total = (
2554 session.query(NoteVersion)
2555 .filter_by(document_id=note_id)
2556 .filter(NoteVersion.change_type.notin_(_AUDIT_BOOKEND_TYPES))
2557 .count()
2558 )
2559 excess = ordinary_total - MAX_VERSIONS_PER_NOTE
2560 if excess > 0:
2561 # Audit bookends have their own cap below and never reduce the
2562 # number of ordinary snapshots retained for the user.
2563 oldest = (
2564 session.query(NoteVersion)
2565 .filter_by(document_id=note_id)
2566 .filter(NoteVersion.change_type.notin_(_AUDIT_BOOKEND_TYPES))
2567 .order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc())
2568 .limit(excess)
2569 .all()
2570 )
2571 for row in oldest:
2572 session.delete(row)
2573 session.flush()
2575 # Separately bound the bookend pool. Excluding bookends from the prune
2576 # above keeps the restore trail, but each restore writes two
2577 # un-prunable rows, so without this ceiling repeated restores grow
2578 # note_versions without limit. Keep the most recent MAX_BOOKEND_VERSIONS.
2579 bookend_total = (
2580 session.query(NoteVersion)
2581 .filter_by(document_id=note_id)
2582 .filter(NoteVersion.change_type.in_(_AUDIT_BOOKEND_TYPES))
2583 .count()
2584 )
2585 bookend_excess = bookend_total - MAX_BOOKEND_VERSIONS
2586 if bookend_excess > 0:
2587 oldest_bookends = (
2588 session.query(NoteVersion)
2589 .filter_by(document_id=note_id)
2590 .filter(NoteVersion.change_type.in_(_AUDIT_BOOKEND_TYPES))
2591 .order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc())
2592 .limit(bookend_excess)
2593 .all()
2594 )
2595 for row in oldest_bookends:
2596 session.delete(row)
2597 session.flush()
2598 ordinary_pruned = max(excess, 0)
2599 bookend_pruned = max(bookend_excess, 0)
2600 return (ordinary_pruned, bookend_pruned)
2602 def _log_version_prune(
2603 self, note_id: str, prune_counts: "tuple[int, int]"
2604 ) -> None:
2605 """Log a prune event. Call ONLY after the caller's transaction has
2606 committed successfully — see ``_prune_versions_in_session`` for why.
2608 Emits aggregate counts only (never note content or titles), and
2609 only when at least one row was actually pruned, so log volume
2610 stays at most one line per save/restore that triggers a prune —
2611 not one line per pruned version.
2612 """
2613 ordinary_pruned, bookend_pruned = prune_counts
2614 if ordinary_pruned or bookend_pruned:
2615 logger.info(
2616 "note version prune: note_id={} ordinary={} bookend={}",
2617 note_id,
2618 ordinary_pruned,
2619 bookend_pruned,
2620 )
2622 def restore_with_bookends(
2623 self, note_id: str, version_id: str
2624 ) -> "tuple[bool, Optional[str]]":
2625 """Restore a note to a previous version atomically.
2627 Wraps PRE_RESTORE snapshot + content update + RESTORE snapshot in
2628 a single transaction. Earlier code did three separate sessions —
2629 process death between writes could leave a restored note with
2630 no audit-trail row. With this method, either all three writes
2631 land or none do.
2633 Returns ``(success, error_code)``:
2634 * ``(True, None)`` — restore complete
2635 * ``(False, "note_not_found")`` — no document with that id, or
2636 it isn't a note
2637 * ``(False, "version_not_found")`` — version id doesn't exist
2638 for this note
2639 * ``(False, "internal_error")`` — exception during restore;
2640 check logs
2642 The route layer maps these to HTTP statuses.
2643 """
2644 try:
2645 with get_user_db_session(self.username) as session:
2646 document = self._get_note_in_session(session, note_id)
2647 if document is None:
2648 return (False, "note_not_found")
2650 version = (
2651 session.query(NoteVersion)
2652 .filter_by(document_id=note_id, id=version_id)
2653 .first()
2654 )
2655 if not version:
2656 return (False, "version_not_found")
2658 # Capture for post-session use (link re-parse)
2659 v_title = version.title
2660 v_content = version.content
2661 v_tags = version.tags
2663 # PRE_RESTORE snapshot of the current document state
2664 self._create_version_snapshot_in_session(
2665 session,
2666 note_id,
2667 document.title,
2668 document.text_content,
2669 document.tags,
2670 NoteChangeType.PRE_RESTORE.value,
2671 )
2673 # Apply the restore to the document
2674 document.title = v_title
2675 document.text_content = v_content
2676 document.tags = v_tags
2677 content_str = v_content or ""
2678 document.character_count = len(content_str)
2679 document.word_count = len(content_str.split())
2680 document.document_hash = self._compute_content_hash(
2681 f"{note_id}:{content_str}"
2682 )
2683 document.file_size = len(content_str.encode("utf-8"))
2685 # RESTORE snapshot marking the post-restore state
2686 self._create_version_snapshot_in_session(
2687 session,
2688 note_id,
2689 v_title,
2690 v_content,
2691 v_tags,
2692 NoteChangeType.RESTORE.value,
2693 change_summary=f"Restored to version {version_id[:8]}",
2694 )
2696 # Prune to cap; with bookends the count can exceed the
2697 # cap by 2 in a single transaction. The deletes are only
2698 # flushed here — this whole transaction can still roll
2699 # back below (stale-reindex or link-reparse failure), so
2700 # the counts are logged only after session.commit()
2701 # succeeds, not here. See _prune_versions_in_session.
2702 prune_counts = self._prune_versions_in_session(session, note_id)
2704 # Stale-vector fix (mirrors update_note): a restore changes
2705 # content/title, so mark its embeddings stale across both
2706 # indexed-state sources — the auto-index worker re-embeds
2707 # and the RAG status report stops showing the pre-restore
2708 # state as indexed. See _mark_note_stale_for_reindex_in_session.
2709 self._mark_note_stale_for_reindex_in_session(session, note_id)
2711 # Re-parse wiki-links INSIDE the restore transaction (against
2712 # the freshly-restored content), exactly like update_note —
2713 # NOT in a separate post-commit session. A post-commit reparse
2714 # raced a concurrent update_note: that save wrote correct
2715 # NoteLink rows for its NEW content inside its own transaction,
2716 # then this stale reparse (computed from the restored content)
2717 # reverted them, leaving the link graph (backlinks/outgoing/
2718 # unlinked-mentions) inconsistent with Document.text_content.
2719 # Trade-off: a reparse failure now rolls the whole restore back
2720 # (the same contract update_note already has) instead of
2721 # logging and leaving a committed-but-mis-linked restore — an
2722 # acceptable, consistent change for link-graph correctness.
2723 self._parse_and_update_links_in_session(
2724 session, note_id, v_content or ""
2725 )
2727 session.commit()
2728 self._log_version_prune(note_id, prune_counts)
2730 return (True, None)
2732 except Exception:
2733 logger.exception(
2734 f"Error restoring version {version_id} for note {note_id}"
2735 )
2736 return (False, "internal_error")
2738 # Note: get_note_versions / get_version / restore_version were removed
2739 # in PR #3277 — the route layer (web/routers/notes.py) reimplements these
2740 # queries directly because they need fine-grained control over the
2741 # session lifetime (e.g., the restore route's PRE_RESTORE/RESTORE
2742 # bookend snapshots). The service-layer versions had zero callers.
2743 # Re-add them only when a non-route caller (CLI, scheduled task)
2744 # actually needs them.
2746 # =========================================================================
2747 # Wiki Linking Methods
2748 # =========================================================================
2750 # NOTE: ``_parse_and_update_links_in_session`` is used by both
2751 # ``create_note`` and ``update_note`` so the content insert/update +
2752 # link rewrite land in a single transaction (atomicity).
2753 def _parse_and_update_links_in_session(
2754 self,
2755 session: Session,
2756 note_id: str,
2757 content: str,
2758 link_overrides: Optional[Dict[str, str]] = None,
2759 ) -> List[Dict[str, Any]]:
2760 """Parse [[wiki-style links]] and update NoteLink rows in an
2761 existing session.
2763 Caller is responsible for committing. Used by ``update_note`` and
2764 ``create_note`` so the content update + link rewrite land in one
2765 transaction — a failure mid-parse rolls the content change back
2766 with it. The standalone ``_parse_and_update_links`` keeps its own
2767 session and is used by ``restore_with_bookends`` which has
2768 explicitly chosen to reparse outside its restore transaction
2769 (documented at the call site).
2771 ``link_overrides`` (lowercased link text → target_document_id) lets
2772 a caller bind a specific ``[[text]]`` to an exact target id.
2773 ``accept_suggested_link`` passes it so a suggestion accepted for one
2774 of several same-titled notes links to THAT note — not whichever the
2775 title resolver happens to pick.
2777 Resolution priority per link: (1) ``link_overrides``; (2) the id this
2778 exact link text resolved to before (the rename-safety cache, now
2779 PREFERRED over a fresh lookup so an accepted/established link stays
2780 id-stable across reparse instead of being silently retargeted to the
2781 oldest same-titled note); (3) a fresh title resolution for brand-new
2782 links. Every id-based target is verified to still be an existing
2783 note before use.
2784 """
2785 # Parse + dedupe targets (first appearance wins) BEFORE applying
2786 # the cap. The cap exists to bound per-save DB work, and duplicate
2787 # texts resolve once regardless — counting each repetition against
2788 # the cap let 1000 repetitions of one ``[[A]]`` silently evict a
2789 # later distinct ``[[B]]``.
2790 parsed_targets: List[str] = []
2791 seen_texts: set[str] = set()
2792 truncated = 0
2793 for raw_link_text in LINK_PATTERN.findall(content):
2794 if len(parsed_targets) >= MAX_LINKS_PER_NOTE:
2795 truncated += 1
2796 continue
2797 # Canonical target text = the portion before the first pipe
2798 # of the Obsidian/Roam ``[[Target|Display]]`` alias syntax.
2799 # The display half is presentation-only; resolution, the
2800 # rename-safety cache key, the stored ``link_text`` and the
2801 # ``resolved_links`` payload all key off the target so the
2802 # frontend linkMap (built from outgoing_links.link_text in
2803 # note-detail.js) matches what ``processWikiLinks`` looks up
2804 # after it splits on the pipe. Storing the raw aliased text
2805 # here would leave every aliased link unresolvable on the
2806 # client and fall back to a slower title search.
2807 target_text, _sep, _display = raw_link_text.partition("|")
2808 target_text = target_text.strip()
2809 if not target_text:
2810 # Empty/whitespace target ("[[ ]]", "[[|alias]]") is not a
2811 # link — the frontend renders it as a literal. Pre-fix the
2812 # raw text was passed through to the resolver, whose
2813 # empty-prefix startswith('') matched EVERY note and
2814 # persisted a phantom NoteLink to the user's oldest note.
2815 continue
2816 text_lc = target_text.lower()
2817 if text_lc in seen_texts:
2818 continue
2819 seen_texts.add(text_lc)
2820 parsed_targets.append(target_text)
2821 if truncated:
2822 logger.warning(
2823 "note {} has more than {} distinct wiki-links; dropping "
2824 "{} matches past the cap to bound per-save DB work",
2825 note_id,
2826 MAX_LINKS_PER_NOTE,
2827 truncated,
2828 )
2830 # Existing rows serve three purposes: the rename-safety cache
2831 # (lowercased link text → target id, so renaming a target doesn't
2832 # nuke the link on the source's next save), the diff base below,
2833 # and preservation of per-row state (id, created_at, and the
2834 # auto_suggested badge from an accepted AI suggestion) on links
2835 # that survive the reparse. If multiple existing rows share the
2836 # same link_text (shouldn't happen — UniqueConstraint is on
2837 # (source, target), not (source, link_text)), the first wins.
2838 existing_rows = (
2839 session.query(NoteLink).filter_by(source_document_id=note_id).all()
2840 )
2841 existing_link_targets: Dict[str, str] = {}
2842 for row in existing_rows:
2843 key = (row.link_text or "").lower().strip()
2844 existing_link_targets.setdefault(key, row.target_document_id)
2845 rows_by_target: Dict[str, NoteLink] = {
2846 row.target_document_id: row for row in existing_rows
2847 }
2849 note_source_type_id = self._get_note_source_type_id(session)
2850 overrides = {
2851 (k or "").lower().strip(): v
2852 for k, v in (link_overrides or {}).items()
2853 }
2855 # Batch-validate every id-based candidate (priorities 1 and 2) in
2856 # one query instead of a per-link primary-key lookup — so a
2857 # stale/deleted/non-note id never resurrects a link, at O(1)
2858 # round-trips instead of O(links).
2859 candidate_ids: set[str] = set()
2860 for target_text in parsed_targets:
2861 text_lc = target_text.lower()
2862 for cid in (
2863 overrides.get(text_lc),
2864 existing_link_targets.get(text_lc),
2865 ):
2866 if cid and cid != note_id:
2867 candidate_ids.add(cid)
2868 valid_targets: Dict[str, str] = {}
2869 candidate_list = sorted(candidate_ids)
2870 for start in range(0, len(candidate_list), _IN_CLAUSE_CHUNK):
2871 chunk = candidate_list[start : start + _IN_CLAUSE_CHUNK]
2872 valid_targets.update(
2873 session.query(Document.id, Document.title)
2874 .filter(
2875 Document.id.in_(chunk),
2876 Document.source_type_id == note_source_type_id,
2877 )
2878 .all()
2879 )
2881 def _valid_note_target(doc_id):
2882 """Return {id, title} if doc_id is an existing note (and not
2883 the source itself), else None."""
2884 if not doc_id or doc_id == note_id:
2885 return None
2886 title = valid_targets.get(doc_id)
2887 return {"id": doc_id, "title": title} if title is not None else None
2889 # Priority-3 pre-pass: batch the exact-title strategy for texts
2890 # priorities 1-2 don't cover. One IN() query over lower(title)
2891 # replaces a per-link table scan; ordering + setdefault implements
2892 # the oldest-wins tie-break (see _resolve_link_internal). The
2893 # mapping back from DB rows to link texts uses _fold_title_ascii,
2894 # which mirrors the database's lower().
2895 fresh_texts = [
2896 t
2897 for t in parsed_targets
2898 if not _valid_note_target(overrides.get(t.lower()))
2899 and not _valid_note_target(existing_link_targets.get(t.lower()))
2900 ]
2901 exact_by_fold: Dict[str, Dict[str, Any]] = {}
2902 folded_list = sorted({_fold_title_ascii(t) for t in fresh_texts})
2903 for start in range(0, len(folded_list), _IN_CLAUSE_CHUNK):
2904 chunk = folded_list[start : start + _IN_CLAUSE_CHUNK]
2905 rows = (
2906 session.query(Document.id, Document.title)
2907 .filter(
2908 Document.source_type_id == note_source_type_id,
2909 func.lower(Document.title).in_(chunk),
2910 )
2911 .order_by(Document.created_at.asc(), Document.id.asc())
2912 .all()
2913 )
2914 for doc_id, title in rows:
2915 exact_by_fold.setdefault(
2916 _fold_title_ascii(title), {"id": doc_id, "title": title}
2917 )
2919 resolved_links: List[Dict[str, Any]] = []
2920 desired: Dict[str, str] = {} # target id → link text (first wins)
2921 prefix_memo: Dict[str, Optional[Dict[str, Any]]] = {}
2922 for target_text in parsed_targets:
2923 text_lc = target_text.lower()
2924 # Priority 1: an explicit override (accept_suggested_link binds
2925 # the exact intended target id, immune to title collisions).
2926 target = _valid_note_target(overrides.get(text_lc))
2927 # Priority 2: the id this exact link text previously resolved to.
2928 # Preferring it over a fresh title lookup keeps an accepted /
2929 # established link pinned to its note across reparse and
2930 # keeps a link alive when its target was renamed.
2931 if not target:
2932 target = _valid_note_target(existing_link_targets.get(text_lc))
2933 # Priority 3: fresh title resolution — batched exact match,
2934 # then the (rare) per-text prefix fallback.
2935 if not target:
2936 fold = _fold_title_ascii(target_text)
2937 target = exact_by_fold.get(fold)
2938 if not target:
2939 if fold not in prefix_memo: 2939 ↛ 2943line 2939 didn't jump to line 2943 because the condition on line 2939 was always true
2940 prefix_memo[fold] = self._resolve_link_prefix(
2941 session, target_text, note_source_type_id
2942 )
2943 target = prefix_memo[fold]
2944 if (
2945 target
2946 and target["id"] != note_id
2947 and target["id"] not in desired
2948 ):
2949 desired[target["id"]] = target_text
2950 resolved_links.append(
2951 {
2952 "link_text": target_text,
2953 "target_id": target["id"],
2954 "target_title": target["title"],
2955 }
2956 )
2958 # Apply as a diff instead of delete-all + recreate: surviving rows
2959 # keep their id, created_at and auto_suggested flag, and an
2960 # unchanged link set writes zero rows.
2961 for target_id, row in rows_by_target.items():
2962 if target_id not in desired:
2963 session.delete(row)
2964 for target_id, link_text in desired.items():
2965 new_text = link_text[:MAX_LINK_TEXT_LENGTH]
2966 row = rows_by_target.get(target_id)
2967 if row is None:
2968 session.add(
2969 NoteLink(
2970 source_document_id=note_id,
2971 target_document_id=target_id,
2972 link_text=new_text,
2973 auto_suggested=False,
2974 )
2975 )
2976 elif row.link_text != new_text: 2976 ↛ 2977line 2976 didn't jump to line 2977 because the condition on line 2976 was never true
2977 row.link_text = new_text
2979 session.flush()
2980 return resolved_links
2982 def _parse_and_update_links(
2983 self, note_id: str, content: str
2984 ) -> List[Dict[str, Any]]:
2985 """Standalone version: opens its own session and commits.
2987 Kept for callers that intentionally manage link reparse outside
2988 their primary transaction (notably ``restore_with_bookends``, where
2989 a link-resolution failure is non-fatal and the restore must not
2990 roll back).
2991 """
2992 try:
2993 with get_user_db_session(self.username) as session:
2994 resolved = self._parse_and_update_links_in_session(
2995 session, note_id, content
2996 )
2997 session.commit()
2998 logger.debug(
2999 f"Updated links for note {note_id}: {len(resolved)} links"
3000 )
3001 return resolved
3002 except Exception:
3003 # Use logger.exception so the stack trace lands in the log.
3004 # Link parsing failures are non-fatal (callers continue), but
3005 # silently dropping the cause makes recurring failures
3006 # invisible to operators.
3007 logger.exception("Failed to parse links for note {}", note_id)
3008 return []
3010 def _resolve_link_internal(
3011 self, session: Session, link_text: str
3012 ) -> Optional[Dict[str, Any]]:
3013 """Resolve a link to a note document.
3015 Supports the Obsidian/Roam pipe alias syntax
3016 ``[[Title|Display Text]]``: the target is everything before the
3017 first ``|``; ``Display Text`` is rendering-only and ignored at
3018 resolution time. Without this split, a link like
3019 ``[[My Note|see here]]`` would search for a literal note titled
3020 "my note|see here" — which never exists — and the link would
3021 silently fall back to a stale cached target id on every save.
3023 Two notes can have the same title (e.g. duplicate "TODO" notes).
3024 Tie-break on ``created_at`` ascending so the link resolves to the
3025 OLDEST match. Resolving to the newest instead reads as more
3026 recency-intuitive, but it silently re-points every existing
3027 ``[[Title]]`` the moment a new note reuses that title (link drift by
3028 spooky-action-at-a-distance); oldest is stable and deterministic
3029 (without the order_by, ``.first()`` is at the DB's discretion and can
3030 flip between calls). Real per-target disambiguation — capturing the
3031 picked note's id at autocomplete time — is tracked as follow-up work.
3032 """
3033 # Strip pipe-alias before lookup. Use partition rather than
3034 # split so a target containing additional pipes (rare but
3035 # legal in markdown) doesn't get further mangled.
3036 target_text, sep, _display = link_text.partition("|")
3037 if sep:
3038 link_text = target_text
3039 link_text = link_text.strip()
3040 # Degenerate target ("[[ ]]" / "[[|alias]]"): an empty key must
3041 # never reach the lookup strategies — a LIKE '%' prefix pattern
3042 # would "resolve" to EVERY note (deterministically the user's
3043 # oldest one). The frontend treats an empty target as a literal,
3044 # not a link; mirror that here.
3045 if not link_text:
3046 return None
3047 source_type_id = self._get_note_source_type_id(session)
3049 # Strategy 1: Exact title match. Fold BOTH sides in SQL: folding
3050 # the link text in Python instead diverges for non-ASCII, because
3051 # SQLite's lower() folds ASCII only while str.lower() folds full
3052 # Unicode — a byte-identical title like "Über Alles" could then
3053 # never match its own link (lower('Über Alles') = 'Über alles',
3054 # but Python sent 'über alles'). SQL-side folding keeps ASCII
3055 # case-insensitivity and makes exact-typed non-ASCII titles
3056 # resolve. (Case-insensitive matching of the non-ASCII letters
3057 # themselves would need a custom collation on the connection.)
3058 document = (
3059 session.query(Document)
3060 .filter(
3061 Document.source_type_id == source_type_id,
3062 func.lower(Document.title) == func.lower(link_text),
3063 )
3064 .order_by(Document.created_at.asc(), Document.id.asc())
3065 .first()
3066 )
3067 if document:
3068 return {"id": document.id, "title": document.title}
3070 return self._resolve_link_prefix(session, link_text, source_type_id)
3072 def _resolve_link_prefix(
3073 self, session: Session, link_text: str, source_type_id: int
3074 ) -> Optional[Dict[str, Any]]:
3075 """Strategy 2: partial (prefix) title match — but only for targets
3076 long enough to be a meaningful prefix. A 1-2 char target like
3077 ``[[i]]`` would otherwise prefix-match and silently persist a
3078 NoteLink to an unrelated note ("Ideas", "Interview notes", ...);
3079 below the minimum length an exact match (Strategy 1) is the only
3080 acceptable resolution. Reuses the same floor as the
3081 unlinked-mention scan.
3083 ``link_text`` must already be stripped. Both sides are folded in
3084 SQL for the same non-ASCII reason as Strategy 1; the pattern is
3085 LIKE-escaped so a target containing wildcards (``[[%]]``,
3086 ``[[_]]``) matches literally instead of over-matching every note.
3087 """
3088 if len(link_text) < self.MIN_MENTION_TITLE_LEN:
3089 return None
3090 prefix_pattern = self._escape_like(link_text) + "%"
3091 document = (
3092 session.query(Document)
3093 .filter(
3094 Document.source_type_id == source_type_id,
3095 func.lower(Document.title).like(
3096 func.lower(prefix_pattern), escape="\\"
3097 ),
3098 )
3099 .order_by(Document.created_at.asc(), Document.id.asc())
3100 .first()
3101 )
3102 if document:
3103 return {"id": document.id, "title": document.title}
3105 return None
3107 # Number of preview chars sent in backlinks / outgoing-links payloads.
3108 # Used by the column-projection SUBSTR so we don't pull full
3109 # text_content (up to 50 MB per Document row) just to slice 200
3110 # chars in Python.
3111 LINK_CONTENT_PREVIEW_CHARS = 200
3113 @staticmethod
3114 def _truncate_preview(raw: Optional[str], preview_len: int) -> str:
3115 """Trim a preview string to ``preview_len`` chars, appending "..."
3116 when the source was longer.
3118 The link-preview queries project ``func.substr(..., 1, preview_len
3119 + 1)`` from the DB — the +1 is the sentinel: SQLite SUBSTR returns
3120 at most preview_len+1 chars, so getting the full +1 char means the
3121 original text_content was longer than preview_len and the preview
3122 should be ellipsized. (SUBSTR uses 1-based indexing via length.)
3123 """
3124 s = raw or ""
3125 return s[:preview_len] + "..." if len(s) > preview_len else s
3127 def get_backlinks(self, note_id: str) -> List[Dict[str, Any]]:
3128 """Get notes that link to the given note.
3130 Errors propagate to the caller so the route layer can return a
3131 proper 500 via ``handle_api_error`` — silently returning ``[]``
3132 on a DB failure would be indistinguishable from "no backlinks"
3133 and mask production issues.
3135 Uses ``func.substr`` to project only the preview prefix of
3136 ``text_content`` from the DB. Pre-fix the JOIN loaded full
3137 Document rows including the unbounded ``text_content`` blob
3138 for every backlink source — at 20 backlinks averaging 100 kB
3139 each, the DB transferred ~2 MB just to slice 4 kB of response.
3140 """
3141 with get_user_db_session(self.username) as session:
3142 preview_len = self.LINK_CONTENT_PREVIEW_CHARS
3143 results = (
3144 session.query(
3145 Document.id,
3146 Document.title,
3147 Document.created_at,
3148 func.substr(
3149 Document.text_content, 1, preview_len + 1
3150 ).label("preview"),
3151 NoteLink.link_text,
3152 )
3153 .join(NoteLink, Document.id == NoteLink.source_document_id)
3154 .filter(NoteLink.target_document_id == note_id)
3155 # Deterministic order so the backlinks panel doesn't reshuffle
3156 # between loads (DB row order is otherwise unspecified).
3157 .order_by(Document.title.asc(), Document.id.asc())
3158 .all()
3159 )
3161 backlinks = []
3162 for row in results:
3163 preview = self._truncate_preview(row.preview, preview_len)
3164 backlinks.append(
3165 {
3166 "id": row.id,
3167 "title": row.title,
3168 "link_text": row.link_text,
3169 "content_preview": preview,
3170 "created_at": row.created_at.isoformat()
3171 if row.created_at
3172 else None,
3173 }
3174 )
3176 return backlinks
3178 def get_outgoing_links(self, note_id: str) -> List[Dict[str, Any]]:
3179 """Get notes that this note links to (outgoing links).
3181 Errors propagate to the caller — see ``get_backlinks`` rationale.
3182 Uses ``func.substr`` column projection for the same reason.
3183 """
3184 with get_user_db_session(self.username) as session:
3185 preview_len = self.LINK_CONTENT_PREVIEW_CHARS
3186 results = (
3187 session.query(
3188 Document.id,
3189 Document.title,
3190 Document.created_at,
3191 func.substr(
3192 Document.text_content, 1, preview_len + 1
3193 ).label("preview"),
3194 NoteLink.link_text,
3195 NoteLink.auto_suggested,
3196 )
3197 .join(NoteLink, Document.id == NoteLink.target_document_id)
3198 .filter(NoteLink.source_document_id == note_id)
3199 # Deterministic order so the outgoing-links panel is stable
3200 # across loads.
3201 .order_by(Document.title.asc(), Document.id.asc())
3202 .all()
3203 )
3205 outgoing = []
3206 for row in results:
3207 preview = self._truncate_preview(row.preview, preview_len)
3208 outgoing.append(
3209 {
3210 "id": row.id,
3211 # Stable FK to the target note. The frontend
3212 # (processWikiLinks) keys its [[link_text]] -> id map on
3213 # target_id to resolve wiki-links by id rather than by a
3214 # fragile title search; without it that lookup was always
3215 # empty and every link fell back to title matching.
3216 "target_id": row.id,
3217 "title": row.title,
3218 "link_text": row.link_text,
3219 "content_preview": preview,
3220 "created_at": row.created_at.isoformat()
3221 if row.created_at
3222 else None,
3223 # True when the link was created by accepting an
3224 # AI suggestion (vs. typed by hand). Lets the UI
3225 # badge AI-suggested links.
3226 "auto_suggested": bool(row.auto_suggested),
3227 }
3228 )
3230 return outgoing
3232 # Titles shorter than this are too generic to mention-match usefully
3233 # (they'd flag almost every note), so unlinked-mention scanning skips them.
3234 MIN_MENTION_TITLE_LEN = 3
3236 def get_unlinked_mentions(
3237 self, note_id: str, limit: int = 20
3238 ) -> List[Dict[str, Any]]:
3239 """Find other notes whose text mentions this note's title but that
3240 don't yet link to it — Obsidian-style "unlinked mentions".
3242 Pure lexical (case-insensitive substring on the title), so it needs no
3243 embeddings/LLM. Notes that already link here are excluded; very short
3244 titles are skipped to avoid matching everything.
3245 """
3246 with get_user_db_session(self.username) as session:
3247 note_source_type_id = self._get_note_source_type_id(session)
3248 if not note_source_type_id: 3248 ↛ 3249line 3248 didn't jump to line 3249 because the condition on line 3248 was never true
3249 return []
3250 note = (
3251 session.query(Document)
3252 .filter_by(id=note_id, source_type_id=note_source_type_id)
3253 .first()
3254 )
3255 title = (note.title or "").strip() if note else ""
3256 if len(title) < self.MIN_MENTION_TITLE_LEN:
3257 return []
3259 # Notes that already link to this note — exclude them.
3260 linked_source_ids = {
3261 row[0]
3262 for row in session.query(NoteLink.source_document_id)
3263 .filter_by(target_document_id=note_id)
3264 .all()
3265 }
3267 # Escape LIKE wildcards in the title so e.g. a "50%" title can't
3268 # match every note.
3269 escaped = self._escape_like(title)
3270 preview_len = self.LINK_CONTENT_PREVIEW_CHARS
3271 query = session.query(
3272 Document.id,
3273 Document.title,
3274 func.substr(Document.text_content, 1, preview_len + 1).label(
3275 "preview"
3276 ),
3277 ).filter(
3278 Document.source_type_id == note_source_type_id,
3279 Document.id != note_id,
3280 Document.text_content.ilike(f"%{escaped}%", escape="\\"),
3281 )
3282 if linked_source_ids:
3283 # Exclude already-linking notes in SQL. Done in Python
3284 # after a ``limit * 3`` over-fetch, a page whose first
3285 # candidates were mostly already linked silently returned
3286 # fewer than ``limit`` mentions even though more existed beyond
3287 # the over-fetch window.
3288 query = query.filter(
3289 Document.id.notin_(list(linked_source_ids))
3290 )
3291 rows = (
3292 # Deterministic order: with no ORDER BY the limited slice was
3293 # whatever order the DB returned, so repeated calls could
3294 # surface different mentions. Newest-updated first, id as
3295 # a stable tiebreaker; mirrors the other link/list panels.
3296 query.order_by(Document.updated_at.desc(), Document.id)
3297 .limit(limit)
3298 .all()
3299 )
3301 return [
3302 {
3303 "id": row.id,
3304 "title": row.title,
3305 "content_preview": self._truncate_preview(
3306 row.preview, preview_len
3307 ),
3308 }
3309 for row in rows
3310 ]
3312 def accept_suggested_link(
3313 self, note_id: str, target_note_id: str
3314 ) -> Optional[Dict[str, Any]]:
3315 """Accept an AI-suggested link by inserting [[Target Title]] into the
3316 note content and creating a NoteLink marked auto_suggested=True.
3318 Returns the updated note dict on success. Returns None only for a
3319 benign precondition that means "nothing to accept" — the note or
3320 target is missing, they are the same note, either is not a note, or
3321 the target title is only wiki-link syntax. A genuine write failure
3322 (e.g. update_note's content-size ValueError, or a DB error) is RAISED,
3323 not collapsed into the same None, so the route can map it to the right
3324 status instead of a misleading 404.
3325 """
3326 try:
3327 with get_user_db_session(self.username) as session:
3328 source = session.query(Document).filter_by(id=note_id).first()
3329 target = (
3330 session.query(Document).filter_by(id=target_note_id).first()
3331 )
3332 if not source or not target:
3333 return None
3335 if source.id == target.id:
3336 logger.warning("Cannot self-link note {}", note_id)
3337 return None
3339 # Check both exist as notes
3340 source_type_id = self._get_note_source_type_id(session)
3341 if ( 3341 ↛ 3345line 3341 didn't jump to line 3345 because the condition on line 3341 was never true
3342 source.source_type_id != source_type_id
3343 or target.source_type_id != source_type_id
3344 ):
3345 return None
3347 # If the note already links to this target (e.g. a hand-typed
3348 # [[Target]]), don't append a duplicate wiki-link or relabel the
3349 # user's link as AI-suggested — accepting is a no-op.
3350 already_linked = (
3351 session.query(NoteLink)
3352 .filter_by(
3353 source_document_id=note_id,
3354 target_document_id=target_note_id,
3355 )
3356 .first()
3357 is not None
3358 )
3359 if already_linked:
3360 return self.get_note(note_id)
3362 # Strip bracket characters from the target title before
3363 # interpolating into the [[...]] wiki-link syntax.
3364 # A title containing "]]" would otherwise corrupt the
3365 # link tokens (e.g. "foo]] [[bar" → "[[foo]] [[bar]]"
3366 # which the parser would split into two separate links).
3367 # Also drop any "|" alias portion: the parser partitions
3368 # the link text on the FIRST pipe and resolves only the
3369 # pre-pipe part, so a title containing "|" would make the
3370 # override key below ("a|b") never match the parser's
3371 # lookup key ("a") — silently bypassing the exact-id
3372 # binding and falling back to title resolution.
3373 # Also collapse internal whitespace (newlines included):
3374 # LINK_PATTERN's character class excludes newlines, so a
3375 # title containing one would append a "[[a\nb]]" literal the
3376 # parser can't match — no NoteLink created, yet the method
3377 # would report success and each retry would append more dead
3378 # text. `" ".join(split())` folds any whitespace run
3379 # (newlines/tabs/multiple spaces) to a single space.
3380 target_title = " ".join(
3381 (target.title or "")
3382 .replace("[", "")
3383 .replace("]", "")
3384 .partition("|")[0]
3385 .split()
3386 )
3387 # Degenerate titles ("|", "[[", empty) strip down to "".
3388 # Appending "[[]]" would persist a dead literal in the
3389 # note body (the parser skips empty link text), so refuse
3390 # instead of silently writing an unlinkable artifact.
3391 # Return None (the method's failure contract — mirrors
3392 # the not-found and self-link branches above).
3393 if not target_title:
3394 logger.warning(
3395 "Cannot accept link {} -> {}: target title contains "
3396 "only wiki-link syntax characters",
3397 note_id,
3398 target_note_id,
3399 )
3400 return None
3402 # Guard the duplicate-title case: if the note already has a
3403 # link whose text matches this title but points to a
3404 # DIFFERENT note (e.g. a hand-typed [[TODO]] resolved to an
3405 # older same-titled note), appending another [[TODO]] would
3406 # not add a second edge — the parser dedupes link text with
3407 # Python str.lower(), so the diff would RETARGET the single
3408 # 'todo' edge to this note and delete the user's existing
3409 # link to the other one. Bare [[title]] can't express two
3410 # different targets for the same text, so refuse rather than
3411 # silently destroy the existing edge.
3412 norm_title = target_title.lower()
3413 sibling_links = (
3414 session.query(NoteLink)
3415 .filter(
3416 NoteLink.source_document_id == note_id,
3417 NoteLink.target_document_id != target_note_id,
3418 )
3419 .all()
3420 )
3421 if any(
3422 (lnk.link_text or "").lower() == norm_title
3423 for lnk in sibling_links
3424 ):
3425 logger.warning(
3426 "Cannot accept link {} -> {}: the note already links "
3427 "'{}' to a different note; bare [[title]] can't "
3428 "disambiguate same-titled targets",
3429 note_id,
3430 target_note_id,
3431 target_title,
3432 )
3433 return None
3435 # Session A above performed only precondition reads. The body
3436 # append itself is NOT computed here from source.text_content:
3437 # that value would be stale by the time update_note's separate
3438 # write transaction runs, so a save landing in between would be
3439 # lost. update_note re-reads the current body and appends inside
3440 # its own transaction (see _append_link_title) — read and write
3441 # are then atomic.
3443 # update_note handles the body append (from the note's CURRENT
3444 # content), versioning, link re-parsing and hash update.
3445 # Bind the appended [[target_title]] to the EXACT target id via an
3446 # override so a title shared by several notes — or a title we had
3447 # to strip brackets from above — links to THIS note, not whichever
3448 # the title resolver would pick. The override id is
3449 # preferred on this parse, and the resulting link's id is then
3450 # preferred on every later reparse (the rename-safety cache), so
3451 # the binding is durable.
3452 self.update_note(
3453 note_id,
3454 _append_link_title=target_title,
3455 _link_overrides={target_title.lower().strip(): target_note_id},
3456 )
3458 # Flag the newly-created NoteLink as auto_suggested=True so UX can
3459 # distinguish it from user-typed links.
3460 with get_user_db_session(self.username) as session:
3461 link = (
3462 session.query(NoteLink)
3463 .filter_by(
3464 source_document_id=note_id,
3465 target_document_id=target_note_id,
3466 )
3467 .first()
3468 )
3469 if link: 3469 ↛ 3473line 3469 didn't jump to line 3473
3470 link.auto_suggested = True
3471 session.commit()
3473 return self.get_note(note_id)
3475 except Exception:
3476 # Real write failures (update_note's ValueError content cap, a DB
3477 # IntegrityError, etc.) must NOT be swallowed into the same None
3478 # the benign preconditions return — that hid genuine failures
3479 # behind a misleading "Link could not be accepted" 404. Log for
3480 # context and re-raise so the route maps it (ValueError -> 400,
3481 # anything else -> 500).
3482 logger.exception(
3483 "Error accepting suggested link {} on {}",
3484 target_note_id,
3485 note_id,
3486 )
3487 raise
3489 def resolve_link(self, link_text: str) -> Optional[Dict[str, Any]]:
3490 """Resolve a [[link]] text to a note.
3492 Public wrapper around ``_resolve_link_internal``. Caps ``link_text``
3493 at ``MAX_LINK_TEXT_LENGTH`` defensively even though ``LINK_PATTERN``
3494 already bounds it — the public route hands its body in untouched.
3495 """
3496 if isinstance(link_text, str) and len(link_text) > MAX_LINK_TEXT_LENGTH:
3497 link_text = link_text[:MAX_LINK_TEXT_LENGTH]
3498 try:
3499 with get_user_db_session(self.username) as session:
3500 return self._resolve_link_internal(session, link_text)
3501 except Exception:
3502 # A DB/session failure must NOT collapse into the same None a
3503 # genuinely unresolvable link returns — the route maps None to a
3504 # 404 "Note not found", which would present an outage as the note
3505 # not existing. Log for context and re-raise so the route's
3506 # handle_api_error surfaces a 500 (same rationale as
3507 # accept_suggested_link above).
3508 logger.exception("Error resolving link")
3509 raise
3511 def search_notes_for_linking(
3512 self, query: str, exclude_note_id: Optional[str] = None, limit: int = 10
3513 ) -> List[Dict[str, Any]]:
3514 """Search notes for the [[link]] autocomplete feature."""
3515 try:
3516 with get_user_db_session(self.username) as session:
3517 source_type_id = self._get_note_source_type_id(session)
3518 search_pattern = f"%{self._escape_like(query)}%"
3520 query_obj = session.query(Document).filter(
3521 Document.source_type_id == source_type_id,
3522 Document.title.ilike(search_pattern, escape="\\"),
3523 )
3525 if exclude_note_id:
3526 query_obj = query_obj.filter(Document.id != exclude_note_id)
3528 documents = (
3529 query_obj.order_by(Document.updated_at.desc())
3530 .limit(limit)
3531 .all()
3532 )
3534 return [
3535 {
3536 "id": doc.id,
3537 "title": doc.title,
3538 "slug": self._generate_slug(doc.title),
3539 }
3540 for doc in documents
3541 ]
3543 except Exception:
3544 # Returning [] here would turn an infrastructure failure into an
3545 # HTTP 200 with an empty autocomplete — indistinguishable from
3546 # "no matching notes". Log for context and re-raise so the route
3547 # surfaces a 500 (matches semantic_search's fix in
3548 # note_ai_service, which removed this exact swallow as masking
3549 # outages).
3550 logger.exception(
3551 "Error searching notes for linking (query_len={})",
3552 len(query) if query else 0,
3553 )
3554 raise