Coverage for src/local_deep_research/research_library/notes/services/note_service.py: 90%
931 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-19 23:35 +0000
1"""
2Note Service - CRUD operations for notes stored as Documents.
4Notes are Documents with source_type='note'. This service provides
5note-specific operations while leveraging the Document infrastructure.
6"""
8import atexit
9import hashlib
10import json
11import random
12import re
13import threading
14import time
15import unicodedata
16import uuid
17from concurrent.futures import ThreadPoolExecutor
18from typing import Any, Dict, List, Optional
20from loguru import logger
21from sqlalchemy import func, or_
22from sqlalchemy.exc import IntegrityError
23from sqlalchemy.orm import Session, joinedload
25from ....database.models import (
26 Collection,
27 Document,
28 DocumentChunk,
29 DocumentCollection,
30 NoteChangeType,
31 NoteLink,
32 NoteReference,
33 NoteResearch,
34 NoteVersion,
35 RagDocumentStatus,
36 SourceType,
37)
38from ....database.session_context import get_user_db_session, safe_rollback
39from ....database.thread_local_session import thread_cleanup
40from ...utils import escape_like
43def escape_markdown_link_label(label: str) -> str:
44 """Escape a string so it is safe as the LABEL of a ``[label](url)``
45 markdown link.
47 Provenance/comment notes embed untrusted text — a research query, or a
48 library document's title (which for downloaded documents comes from
49 third-party HTML/PDF metadata) — as the label of an auto-generated
50 link. Without escaping, a label like ``evil](https://attacker/x`` ends
51 the label at its first ``]`` and lets the rest re-target the rendered
52 link at an attacker URL (a phishing/content-integrity issue when the
53 note is later rendered as markdown, and a syntax-injection risk in the
54 PDF/LaTeX/Quarto exporters). Escape the four link-structural
55 metacharacters (backslash first) and neutralise newlines so the label
56 can never break out of the brackets.
57 """
58 return (
59 str(label)
60 .replace("\\", "\\\\")
61 .replace("[", "\\[")
62 .replace("]", "\\]")
63 .replace("(", "\\(")
64 .replace(")", "\\)")
65 .replace("\r", " ")
66 .replace("\n", " ")
67 )
70def _capture_request_db_password(username: str) -> Optional[str]:
71 """Capture the encrypted-DB password from the request thread.
73 Thin wrapper over the shared
74 ``database.session_passwords.capture_request_db_password`` (kept as a
75 module-local name so the many call sites in this file stay unchanged).
76 Needed because ``ThreadPoolExecutor.submit`` does not copy the calling
77 thread's ``ContextVar`` snapshot, so background workers otherwise lose
78 the password and every encrypted-DB write is swallowed.
79 """
80 from ....database.session_passwords import capture_request_db_password
82 return capture_request_db_password(username)
85# Bounded thread pool for AI change-summary generation. summarize_changes is
86# the only call inside update_note that hits a remote LLM, so leaving it on
87# the request thread pays a full round-trip on every content-changing save.
88# We persist the version snapshot synchronously with change_summary=None,
89# then enqueue an UPDATE that fills the summary once the LLM returns. Users
90# see their save reflected in history immediately; the summary populates
91# moments later when they open the version panel.
92_summary_executor: ThreadPoolExecutor | None = None
93_summary_executor_lock = threading.Lock()
96def _get_summary_executor() -> ThreadPoolExecutor:
97 global _summary_executor
98 with _summary_executor_lock:
99 if _summary_executor is None:
100 _summary_executor = ThreadPoolExecutor(
101 max_workers=4,
102 thread_name_prefix="note_summary_",
103 )
104 return _summary_executor
107# Soft cap on pending change-summary tasks. ThreadPoolExecutor's
108# internal queue is unbounded; combined with the per-task (old_content,
109# new_content) capture (up to 50 MB each), a scripted burst of saves
110# could pile up hundreds of MB before the 4 workers drain it. Drop the
111# task with a log line once the queue exceeds the cap — losing the AI
112# change summary is a non-essential degradation; the snapshot row
113# itself already landed at update_note commit time. Per-user write
114# rate-limit (``_notes_write_limit`` at 60/min) provides the primary
115# defence; this is belt-and-suspenders for unexpected hot paths.
116_SUMMARY_QUEUE_SOFT_CAP = 100
119def _submit_summary_task(fn, *args, **kwargs) -> None:
120 """Submit a change-summary task with queue-depth backpressure.
122 On overload, drops the task and logs at WARNING level — better than
123 silent unbounded queue growth. The snapshot row is unaffected;
124 change_summary stays NULL until the next save fills it.
125 """
126 executor = _get_summary_executor()
127 # ThreadPoolExecutor exposes its work queue via the private
128 # ``_work_queue`` attribute. The qsize() reading is approximate
129 # (queue is concurrent-safe but the value can be stale by the
130 # time we read it) but the soft cap intentionally tolerates drift.
131 try:
132 qsize = executor._work_queue.qsize()
133 except Exception:
134 qsize = 0
135 if qsize >= _SUMMARY_QUEUE_SOFT_CAP:
136 logger.warning(
137 "Summary executor queue at {} items (cap {}); dropping task. "
138 "NoteVersion.change_summary will stay NULL for this save.",
139 qsize,
140 _SUMMARY_QUEUE_SOFT_CAP,
141 )
142 return
143 # Wrap in thread_cleanup so the worker clears its thread-local DB
144 # session AND the cached (username, password) entry in
145 # _thread_credentials when it returns. The summary pool's threads are
146 # long-lived (they never die), so cleanup_dead_threads never sweeps
147 # them — without this the plaintext SQLCipher password the worker
148 # captured would sit in the pool thread's dict for the whole process
149 # lifetime (the #4182 credential-hygiene class). Mirrors the idiom
150 # every other executor in the codebase uses (e.g. news_strategy).
151 try:
152 executor.submit(thread_cleanup(fn), *args, **kwargs)
153 except RuntimeError:
154 # The pool was shut down (graceful restart / atexit) between the
155 # _get_summary_executor() read above and this submit. change_summary is
156 # best-effort, so drop it rather than let the RuntimeError bubble up and
157 # 500 a note save whose content already committed durably.
158 logger.warning(
159 "Summary executor shut down before task submit; "
160 "NoteVersion.change_summary will stay NULL for this save."
161 )
164def _shutdown_summary_executor() -> None:
165 # Race fix: the matching getter at _get_summary_executor holds the
166 # lock for its read-modify-write of _summary_executor; the shutdown
167 # path must hold the same lock or a request thread can observe a
168 # half-replaced reference (e.g. submit() into an already-shut-down
169 # executor → RuntimeError) during gunicorn graceful shutdown.
170 global _summary_executor
171 with _summary_executor_lock:
172 if _summary_executor is not None:
173 _summary_executor.shutdown(wait=True)
174 _summary_executor = None
177atexit.register(_shutdown_summary_executor)
180def _populate_change_summary_async(
181 username: str,
182 dbpw: Optional[str],
183 version_id: str,
184 old_content: str,
185 new_content: str,
186) -> None:
187 """Background worker: ask the LLM for a change summary and UPDATE the
188 already-persisted NoteVersion row.
190 ``dbpw`` is the encrypted-DB password captured on the request thread by
191 ``_capture_request_db_password`` and threaded in explicitly because
192 stdlib ``ThreadPoolExecutor`` does not propagate the request's
193 ``ContextVar`` snapshot — without it, ``get_user_db_session`` raises
194 ``DatabaseSessionError`` on every encrypted-DB install. It's named
195 ``dbpw`` rather than ``db_password`` to avoid a gitleaks generic-secret
196 false positive (that rule matches a ``password=<8+ char literal>`` shape);
197 the value is a passthrough credential, never a hardcoded secret.
199 Exceptions are logged but never raised — this runs on a thread pool and
200 a failure here must not crash the worker.
201 """
202 try:
203 from .note_ai_service import NoteAIService
205 # Pass dbpw so the LLM-settings lookup inside summarize_changes ->
206 # _get_llm() can open the encrypted DB on this worker thread (no
207 # request context to resolve the password from).
208 ai_service = NoteAIService(username, dbpw=dbpw)
209 summary = ai_service.summarize_changes(old_content, new_content)
210 if not summary: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true
211 return
213 with get_user_db_session(username, password=dbpw) as session:
214 row = session.query(NoteVersion).filter_by(id=version_id).first()
215 if row is None: 215 ↛ 216line 215 didn't jump to line 216 because the condition on line 215 was never true
216 logger.debug(
217 "Version {} disappeared before summary landed; skipping",
218 version_id[:8],
219 )
220 return
221 # Fill-only guard: this worker is only ever scheduled for a
222 # freshly inserted snapshot (change_summary=None). Refuse to
223 # overwrite an existing summary so a caller bug (e.g. passing
224 # a dedup-absorbed historical version id) can't corrupt the
225 # audit trail.
226 if row.change_summary is not None:
227 logger.warning(
228 "Version {} already has a change summary; refusing to "
229 "overwrite it",
230 version_id[:8],
231 )
232 return
233 row.change_summary = summary
234 session.commit()
235 logger.debug(
236 "Filled change summary for note version {}", version_id[:8]
237 )
238 except Exception:
239 logger.exception(
240 "Async change-summary worker failed for version {}",
241 version_id[:8] if version_id else "<none>",
242 )
245# Regex for parsing [[wiki-style links]]. The negated class is bounded by
246# MAX_LINK_TEXT_LENGTH so an unclosed ``[[`` doesn't capture multi-MB of
247# body and ship it as a SQL parameter twice (exact + startswith) against
248# the unindexed LOWER(title) column. It also excludes newlines (``\n``): a
249# note title is a single-line String(500), so a ``[[`` whose closing
250# ``]]`` is on a later line is an unclosed bracket, not a link. This
251# mirrors the frontend's ``WIKILINK_TARGET = [^\]\n]+`` in
252# web/static/js/services/formatting.js so parse (which creates NoteLink
253# rows) and render (which draws the link) agree — previously the backend
254# admitted newlines and created links the UI never rendered.
255MAX_LINK_TEXT_LENGTH = 500
256LINK_PATTERN = re.compile(r"\[\[([^\]\n]{1,500})\]\]")
258# Cap on parsed wiki-links per note. Without this, a pathological note
259# (e.g. 50 MB of `[[A]]\n` lines = millions of matches) would spawn
260# millions of NoteLink rows on save and have each one trigger 2-3
261# database queries through ``_resolve_link_internal``. 1000 covers any
262# realistic note (Roam/Obsidian power users typically have <100 links
263# per note); anything beyond is silently truncated and a warning logged
264# rather than failing the save.
265MAX_LINKS_PER_NOTE = 1000
267# Mirrors SQLite's built-in lower(): ASCII-only case folding. Link
268# resolution folds titles in SQL, and the batched exact-title lookup has
269# to match lower(title) values computed BY THE DATABASE back to
270# Python-side link texts — so the Python fold must be the database's.
271# str.lower() folds full Unicode ('Ü' → 'ü') and would diverge.
272_SQLITE_LOWER_TABLE = str.maketrans(
273 "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
274)
277def _fold_title_ascii(text: str) -> str:
278 return (text or "").translate(_SQLITE_LOWER_TABLE)
281# SQLITE_MAX_VARIABLE_NUMBER is 999 on older SQLite builds; chunk IN()
282# lists well below it so a 1000-link note can't overflow the bind-param
283# limit.
284_IN_CLAUSE_CHUNK = 500
286# Caps on per-note many-to-many list params accepted by the route
287# layer. Mirror MAX_TAGS_PER_NOTE in spirit: bound how many DocumentCollection
288# / NoteResearch rows a single create or reorder request can spawn.
289MAX_COLLECTIONS_PER_NOTE = 50
290MAX_RESEARCH_PER_NOTE = 200
292# Mirrors BaseExporter.MAX_CONTENT_SIZE at exporters/base.py:63. Kept inline
293# rather than imported because exporters/__init__.py unconditionally pulls in
294# weasyprint via all 5 exporter modules — importing BaseExporter would drag a
295# heavy dependency into note CRUD. Extract to a dep-light constants module if
296# sharing ever becomes worthwhile.
297NOTE_CONTENT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
299# Cap on version-history rows per note. Auto-save can otherwise grow
300# note_versions unboundedly — a near-50 MB note with small changes x 100 saves
301# = ~5 GB. Pure FIFO: simplest rule with useful value; the `initial` version
302# can be pruned, but the user still has 100 recent snapshots to fall back on.
303MAX_VERSIONS_PER_NOTE = 100
304# Audit bookends (PRE_RESTORE / RESTORE) are excluded from the normal version
305# prune so the "restored here" trail survives heavy editing — but they still
306# need a ceiling, or repeated restores grow note_versions without bound. Keep
307# the most recent N bookends.
308MAX_BOOKEND_VERSIONS = 40
310# PRE_RESTORE / RESTORE are the audit bookends excluded from version-dedup
311# (their hashes are salted) and from the prune pool (the "restored here" trail
312# survives heavy editing).
313_AUDIT_BOOKEND_TYPES = (
314 NoteChangeType.PRE_RESTORE.value,
315 NoteChangeType.RESTORE.value,
316)
319def _assert_content_size(content: str) -> None:
320 """Raise ValueError if content isn't a string or exceeds
321 NOTE_CONTENT_MAX_BYTES.
323 The type check matters for direct API callers: a truthy non-string
324 body value (e.g. ``{"content": 123}``) passes the route layer's
325 ``if not content`` guard, and without this check ``.encode()`` raised
326 an AttributeError that surfaced as an opaque 500 instead of the 400
327 the routes' ``except ValueError`` mapping produces.
328 """
329 if not isinstance(content, str):
330 raise ValueError("content must be a string")
331 if content and len(content.encode("utf-8")) > NOTE_CONTENT_MAX_BYTES:
332 raise ValueError(
333 f"Note content exceeds maximum size "
334 f"({NOTE_CONTENT_MAX_BYTES // (1024 * 1024)} MB)"
335 )
338# Mirrors MAX_TAG_LENGTH in note-detail.js. The frontend caps the input
339# field, but direct API callers can still submit arbitrary tag strings.
340# Backend validation is the only stop against a 10MB tag landing in the
341# JSON ``tags`` column.
342MAX_TAG_LENGTH = 50
344# Cap the number of tags per note. Without this, ``_validate_tags`` accepts
345# an unbounded list — the 50-char per-tag limit then permits multi-MB of
346# tag JSON per note, and snapshots store the tag list per version, so the
347# 100-version cap multiplies the bloat.
348MAX_TAGS_PER_NOTE = 50
351def _validate_tags(tags: Optional[List[str]]) -> None:
352 """Raise ValueError if ``tags`` isn't a list of short strings.
354 None passes (callers may pass None to mean "don't update tags"). An
355 empty list passes. Anything else: must be a list, every entry must
356 be a string, and each string must be ``<= MAX_TAG_LENGTH`` chars.
357 """
358 if tags is None:
359 return
360 if not isinstance(tags, list):
361 raise ValueError("tags must be a list")
362 if len(tags) > MAX_TAGS_PER_NOTE:
363 raise ValueError(f"too many tags (max {MAX_TAGS_PER_NOTE})")
364 for tag in tags:
365 if not isinstance(tag, str):
366 raise ValueError("each tag must be a string")
367 if len(tag) > MAX_TAG_LENGTH:
368 raise ValueError(
369 f"tag exceeds maximum length ({MAX_TAG_LENGTH} chars)"
370 )
373# Cap on note title length. ``Document.title`` is a plain ``Text`` column
374# (unbounded), but the title is interpolated raw into LLM synthesis prompts
375# (note_ai_service.py:729) and into ``LOWER(title) == ?`` wiki-link lookups
376# (note_service._resolve_link_internal) on every save. Without a cap a
377# direct API caller could send a 50 MB title and DoS both paths.
378MAX_TITLE_LENGTH = 500
381def _validate_title(title: Optional[str]) -> None:
382 """Raise ValueError if ``title`` isn't a short non-blank string.
384 None passes (callers may pass None to "don't update title"). Empty
385 string passes (the route layer's own ``if not title`` check still
386 rejects it on create). Whitespace-only strings (``" "``,
387 ``"\\t\\n"``) are explicitly rejected so the slug generator does not
388 produce an empty slug and the wiki-link resolver doesn't match an
389 invisible title via ``[[ ]]``. Anything else: must be a string and
390 at most ``MAX_TITLE_LENGTH`` characters.
391 """
392 if title is None:
393 return
394 if not isinstance(title, str): 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 raise ValueError("title must be a string")
396 # Reject whitespace-only titles. Empty string falls through to the
397 # route-layer guard; whitespace-only is truthy and would slip past
398 # ``if not title`` checks.
399 if title and not title.strip():
400 raise ValueError("title must not be blank or whitespace-only")
401 if len(title) > MAX_TITLE_LENGTH:
402 raise ValueError(
403 f"title exceeds maximum length ({MAX_TITLE_LENGTH} chars)"
404 )
407class NoteService:
408 """Service for managing notes (Documents with source_type='note')."""
410 DEFAULT_NOTES_COLLECTION_NAME = "Notes"
411 SOURCE_TYPE_NAME = "note"
413 def __init__(self, username: str):
414 """Initialize note service for a user."""
415 self.username = username
417 @staticmethod
418 def _generate_slug(title: str) -> str:
419 """Generate a URL-friendly slug from a title.
421 Folds Latin accents to ASCII via stdlib NFKD normalization
422 (``café`` → ``cafe``). Scripts with no ASCII form (CJK, Cyrillic,
423 Arabic) and emoji drop out, so a title that reduces to nothing falls
424 back to the ``"note"`` placeholder. A real transliterator is avoided
425 on purpose: the licensed options (text-unidecode / python-slugify)
426 are copyleft/GPL, and the slug is a non-critical convenience field.
427 """
428 ascii_title = (
429 unicodedata.normalize("NFKD", title or "")
430 .encode("ascii", "ignore")
431 .decode("ascii")
432 )
433 slug = ascii_title.lower()
434 slug = re.sub(r"\s+", "-", slug)
435 slug = re.sub(r"[^a-z0-9\-]", "", slug)
436 slug = re.sub(r"-+", "-", slug)
437 slug = slug.strip("-")
438 return (slug or "note")[:500]
440 @staticmethod
441 def _compute_content_hash(content: str) -> str:
442 """Compute SHA-256 hash of content."""
443 return hashlib.sha256(content.encode("utf-8")).hexdigest()
445 @staticmethod
446 def _compute_version_hash(
447 title: Optional[str],
448 content: Optional[str],
449 tags: Optional[List[str]],
450 ) -> str:
451 """Compute the dedup hash for a NoteVersion snapshot.
453 JSON-encodes the whole ``[title, content, sorted-tags]`` triple and
454 hashes that, so title-only / content-only / tag-only edits each get a
455 distinct hash and aren't silently deduped against an existing snapshot.
456 Tags are sorted first so reordering alone doesn't churn the hash.
458 Why JSON the WHOLE triple rather than a NUL-joined string: title and
459 content can each contain a literal NUL byte — a JSON ``\\u0000`` in the
460 request body decodes to ``\\x00`` and survives SQLite TEXT storage
461 untouched (SQLite is length-prefixed, not NUL-terminated), and nothing
462 validates it away — so a NUL separator is NOT injection-proof. Pre-fix,
463 ``title="MyTitle", content="Hello\\x00World"`` and
464 ``title="MyTitle\\x00Hello", content="World"`` produced the identical
465 NUL-joined payload and hash, silently dedup-absorbing a genuinely
466 different save out of version history. JSON escapes every control
467 character (NUL included) as ``\\uXXXX`` and quotes/escapes each element,
468 making the encoding injective across all three fields.
470 Hash-format note: this serialization differs from older builds', so the
471 hash changes for previously-written snapshots. The only consumer is the
472 (document_id, content_hash) dedup check, so the sole effect is a
473 one-time extra version row the first time a note is re-saved in a state
474 that already has an old-format snapshot — benign.
475 """
476 title_s = "" if title is None else title
477 content_s = "" if content is None else content
478 sorted_tags = sorted(t for t in (tags or []) if isinstance(t, str))
479 payload = json.dumps(
480 [title_s, content_s, sorted_tags], ensure_ascii=False
481 )
482 return hashlib.sha256(payload.encode("utf-8")).hexdigest()
484 def _get_note_source_type_id(self, session: Session) -> str:
485 """Get the source_type ID for notes, creating it if needed."""
486 from . import get_note_source_type_id
488 source_type_id = get_note_source_type_id(session)
489 if source_type_id: 489 ↛ 493line 489 didn't jump to line 493 because the condition on line 489 was always true
490 return source_type_id
492 # Create it if it doesn't exist (shouldn't happen normally)
493 source_type = SourceType(
494 id=str(uuid.uuid4()),
495 name=self.SOURCE_TYPE_NAME,
496 display_name="Note",
497 description="User-created notes with AI-enhanced features",
498 icon="sticky-note",
499 )
500 session.add(source_type)
501 session.flush()
502 return source_type.id
504 def _get_or_create_notes_collection(self, session: Session) -> str:
505 """Get or create the default Notes collection.
507 Serialised by the per-user init lock to prevent two concurrent
508 first-note creations (e.g. two browser tabs both creating their
509 first note simultaneously) from each seeing no ``collection_type='notes'``
510 row and inserting a duplicate. Matches the locking pattern used
511 by ``ensure_default_library_collection`` and
512 ``ensure_research_history_collection`` in ``database/library_init.py``;
513 there is no UNIQUE constraint at the schema level that would
514 catch the race otherwise.
516 The create path COMMITS inside the lock, for two reasons (both
517 matching the ``library_init`` reference pattern):
519 1. Lock scope: a flush-only insert leaves the row invisible to
520 other sessions until the caller commits — after the lock is
521 released — so a second request entering the lock in that window
522 would still see no row and insert a duplicate.
523 2. Read-only callers: routes like ask-context reach this via
524 ``get_notes_collection_id`` and never commit; the request
525 teardown ROLLS BACK the shared session, discarding the row
526 after its id was already returned to the client.
528 Because the commit flushes the caller's whole session, callers
529 must resolve the collection BEFORE staging unrelated writes on
530 the same session (create_note does).
531 """
532 from ....database.library_init import _get_user_init_lock
534 with _get_user_init_lock(self.username):
535 collection = (
536 session.query(Collection)
537 .filter_by(collection_type="notes")
538 .first()
539 )
541 if collection:
542 return collection.id
544 collection_id = str(uuid.uuid4())
545 collection = Collection(
546 id=collection_id,
547 name=self.DEFAULT_NOTES_COLLECTION_NAME,
548 description="Default collection for notes",
549 collection_type="notes",
550 )
551 session.add(collection)
552 try:
553 session.commit()
554 except Exception:
555 session.rollback()
556 raise
557 logger.info("Created default Notes collection: {}", collection_id)
558 return collection_id
560 def get_notes_collection_id(self) -> Optional[str]:
561 """Get the default Notes collection ID.
563 Returns the collection ID or None if it cannot be determined.
564 """
565 try:
566 with get_user_db_session(self.username) as session:
567 return self._get_or_create_notes_collection(session)
568 except Exception:
569 logger.exception("Error getting notes collection ID")
570 return None
572 def get_notes_index_status(self) -> Dict[str, Any]:
573 """Pre-flight context for "Ask your notes": the Notes collection id,
574 the total note count, and how many notes are indexed into it.
576 The UI uses this to warn when there is nothing to search — no notes
577 at all, or notes that haven't been embedded yet — before kicking off
578 a research run pinned to the Notes collection.
579 """
580 collection_id = self.get_notes_collection_id()
581 note_count = self.count_notes()
582 indexed_count = 0
583 if collection_id: 583 ↛ 590line 583 didn't jump to line 590 because the condition on line 583 was always true
584 with get_user_db_session(self.username) as session:
585 indexed_count = (
586 session.query(DocumentCollection)
587 .filter_by(collection_id=collection_id, indexed=True)
588 .count()
589 )
590 return {
591 "collection_id": collection_id,
592 "note_count": note_count,
593 "indexed_count": indexed_count,
594 }
596 def _is_note(self, session: Session, document: Document) -> bool:
597 """Check if a document is a note."""
598 source_type = (
599 session.query(SourceType)
600 .filter_by(id=document.source_type_id)
601 .first()
602 )
603 return source_type and source_type.name == self.SOURCE_TYPE_NAME
605 def _get_note_in_session(
606 self, session: Session, note_id: str
607 ) -> Optional[Document]:
608 """Fetch a Document by id and return it only if it is a note.
610 Centralizes the fetch + ``_is_note`` guard prelude that every
611 mutating note method hand-copied — and which was half-copied as a
612 bug on the collection helpers (they originally skipped the
613 ``_is_note`` check, letting a non-note Document acquire note rows).
614 Returns ``None`` when the id doesn't exist or the Document isn't a
615 note; callers map that to their own bail value.
617 Still routes through ``self._is_note`` so the existing
618 ``patch.object(NoteService, '_is_note')`` monkeypatches keep firing.
619 """
620 document = session.query(Document).filter_by(id=note_id).first()
621 if document is None or not self._is_note(session, document):
622 return None
623 return document
625 @staticmethod
626 def _escape_like(text: str) -> str:
627 """Escape LIKE/ILIKE wildcards so user input matches literally.
629 Thin alias for the shared :func:`escape_like` so every notes query
630 path uses the one canonical escaping rule.
631 """
632 return escape_like(text)
634 @staticmethod
635 def _rollback_quietly(session) -> None:
636 """Roll back a session after a failed flush/commit, swallowing any
637 rollback error. The per-user request session is shared and is NOT
638 rolled back by get_user_db_session on exception, so callers that catch
639 IntegrityError (and want to retry or surface a clean error) must roll
640 back explicitly or the next statement reuses a poisoned session.
642 Keeps the None-guard (call sites pass ``locals().get("session")``,
643 which may be None) and delegates the actual rollback/log to the
644 shared :func:`safe_rollback` helper.
645 """
646 if session is None: 646 ↛ 647line 646 didn't jump to line 647 because the condition on line 646 was never true
647 return
648 safe_rollback(session, "note_service")
650 def create_note(
651 self,
652 title: str,
653 content: str,
654 tags: Optional[List[str]] = None,
655 collection_ids: Optional[List[str]] = None,
656 ) -> str:
657 """
658 Create a new note (stored as a Document).
660 Args:
661 title: Note title
662 content: Note content (markdown)
663 tags: Optional list of tags
664 collection_ids: Optional list of collection IDs
666 Returns:
667 The created note/document ID
668 """
669 _validate_title(title)
670 _assert_content_size(content)
671 _validate_tags(tags)
672 if collection_ids is not None:
673 if not isinstance(collection_ids, list): 673 ↛ 674line 673 didn't jump to line 674 because the condition on line 673 was never true
674 raise ValueError("collection_ids must be a list")
675 if len(collection_ids) > MAX_COLLECTIONS_PER_NOTE:
676 raise ValueError(
677 f"too many collection_ids (max {MAX_COLLECTIONS_PER_NOTE})"
678 )
679 # Element type check: an unhashable element (dict/list) would
680 # otherwise raise TypeError in the de-dupe set comprehension
681 # below — an opaque 500 instead of the routes' ValueError→400.
682 for cid in collection_ids:
683 if not isinstance(cid, str):
684 raise ValueError("each collection_id must be a string")
685 try:
686 with get_user_db_session(self.username) as session:
687 note_id = str(uuid.uuid4())
688 source_type_id = self._get_note_source_type_id(session)
689 content_hash = self._compute_content_hash(
690 f"{note_id}:{content}"
691 )
693 # Resolve the default Notes collection BEFORE staging the
694 # note: the get-or-create commits when it has to create the
695 # collection (see its docstring), and committing after
696 # session.add(document) would break this method's
697 # one-transaction atomicity for the note itself. A lazily
698 # created SourceType (flushed above) riding along on that
699 # commit is fine — it's the same kind of idempotent init
700 # row as the collection.
701 default_collection_id = self._get_or_create_notes_collection(
702 session
703 )
705 # Create Document with source_type='note'
706 document = Document(
707 id=note_id,
708 source_type_id=source_type_id,
709 document_hash=content_hash,
710 file_size=len(content.encode("utf-8")),
711 file_type="note",
712 title=title,
713 text_content=content,
714 tags=tags or [],
715 # DB column stays `favorite` (the flag is shared with the
716 # Document model); the notes API/UI vocabulary for it is
717 # `pinned` (see the `"pinned": document.favorite` response
718 # mapping). One column, two names by design.
719 favorite=False,
720 character_count=len(content),
721 word_count=len(content.split()),
722 )
723 session.add(document)
725 # Add to default Notes collection
726 doc_coll = DocumentCollection(
727 document_id=note_id,
728 collection_id=default_collection_id,
729 indexed=False,
730 )
731 session.add(doc_coll)
733 # Add to additional collections. De-dupe and skip ids that
734 # don't exist: a repeated id would violate the
735 # (document_id, collection_id) UNIQUE and a bogus id would
736 # FK-violate — either one aborting the entire note creation
737 # with an opaque error.
738 if collection_ids:
739 requested = {cid for cid in collection_ids if cid} - {
740 default_collection_id
741 }
742 valid_ids = {
743 row[0]
744 for row in session.query(Collection.id).filter(
745 Collection.id.in_(requested)
746 )
747 }
748 for coll_id in valid_ids:
749 session.add(
750 DocumentCollection(
751 document_id=note_id,
752 collection_id=coll_id,
753 indexed=False,
754 )
755 )
757 # Atomicity: Document + DocumentCollection + INITIAL
758 # version snapshot + outgoing links all land in one
759 # transaction. Pre-fix, the document committed first and
760 # the snapshot + link reparse ran in fresh sessions, so a
761 # crash between writes left a note with no INITIAL audit
762 # row and no parsed [[wiki-links]] — same shape as the
763 # bug that update_note already fixed.
764 self._create_version_snapshot_in_session(
765 session,
766 note_id=note_id,
767 title=title,
768 content=content,
769 tags=tags,
770 change_type=NoteChangeType.INITIAL.value,
771 change_summary="Initial version",
772 )
773 self._parse_and_update_links_in_session(
774 session, note_id, content
775 )
777 try:
778 session.commit()
779 except Exception:
780 session.rollback()
781 raise
783 # Don't log the title — note titles can carry sensitive
784 # content (e.g. medical, personal). Note id alone is
785 # enough to correlate with surrounding events.
786 logger.info("Created note {}", note_id)
788 return note_id
790 except Exception:
791 # Same atomicity guarantee as update_note: the cached
792 # request/thread session does NOT auto-rollback. The
793 # in-session helpers (snapshot, link reparse) call
794 # session.flush() at the end, so a downstream failure leaves
795 # the document INSERT durably flushed without ever committing
796 # properly. Rollback explicitly so create_note is atomic.
797 # locals().get avoids NameError when the failure happens before
798 # `with ... as session` binds; _rollback_quietly no-ops on None.
799 self._rollback_quietly(locals().get("session"))
800 logger.exception("Error creating note")
801 raise
803 def note_exists(self, note_id: str) -> bool:
804 """Cheap existence + is-note guard for route-level 404 checks.
806 Projects only ``Document.id`` filtered by the note source type —
807 no ``text_content`` hydration (a plain non-deferred column, up to
808 50 MB per row via ``get_note``) and none of ``get_note``'s
809 aggregate side queries. Use this wherever the caller only needs
810 to know the note is real; ``get_note`` stays for handlers that
811 consume the payload.
812 """
813 if not note_id: 813 ↛ 814line 813 didn't jump to line 814 because the condition on line 813 was never true
814 return False
815 with get_user_db_session(self.username) as session:
816 note_source_type_id = self._get_note_source_type_id(session)
817 if note_source_type_id is None: 817 ↛ 818line 817 didn't jump to line 818 because the condition on line 817 was never true
818 return False
819 return (
820 session.query(Document.id)
821 .filter(
822 Document.id == note_id,
823 Document.source_type_id == note_source_type_id,
824 )
825 .first()
826 is not None
827 )
829 def get_note(self, note_id: str) -> Optional[Dict[str, Any]]:
830 """Get a note by ID.
832 The response includes an ``outgoing_links`` map (link_text →
833 target_id, target_title) so the frontend can resolve
834 ``[[wiki-links]]`` to the correct target Document at render
835 time, regardless of whether the target has since been renamed.
836 Pre-fix the frontend resolved by title text alone via
837 ``navigateToNoteByTitle``, so renaming a linked note silently
838 broke every existing ``[[OldTitle]]`` in other notes' content.
839 """
840 try:
841 with get_user_db_session(self.username) as session:
842 document = self._get_note_in_session(session, note_id)
843 if document is None:
844 return None
846 result = self._document_to_note_dict(document, session)
848 # Outgoing-link resolution table for the frontend wiki-link
849 # renderer. Keep this lean (only the link_text and the
850 # target_id/title) — the full backlinks payload is served
851 # by GET /api/notes/<id>/outgoing-links on demand.
852 outgoing_links = (
853 session.query(
854 NoteLink.link_text,
855 Document.id,
856 Document.title,
857 )
858 .join(NoteLink, Document.id == NoteLink.target_document_id)
859 .filter(NoteLink.source_document_id == note_id)
860 .all()
861 )
862 result["outgoing_links"] = [
863 {
864 "link_text": row.link_text,
865 "target_id": row.id,
866 "target_title": row.title,
867 }
868 for row in outgoing_links
869 ]
870 return result
872 except Exception:
873 logger.exception("Error getting note {}", note_id)
874 raise
876 def update_note(
877 self,
878 note_id: str,
879 title: Optional[str] = None,
880 content: Optional[str] = None,
881 tags: Optional[List[str]] = None,
882 favorite: Optional[bool] = None,
883 _link_overrides: Optional[Dict[str, str]] = None,
884 _append_link_title: Optional[str] = None,
885 ) -> bool:
886 """Update a note.
888 Naming note: ``favorite`` matches ``Document.favorite`` (the column
889 backing this state). The user-facing UI vocabulary is "pin"; the
890 wire format keeps ``pinned`` for that reason and the route layer
891 translates. Don't rename the kwarg back without also updating the
892 Document column.
894 ``_append_link_title`` (internal, used by ``accept_suggested_link``):
895 when set, the new body is derived from the note's CURRENT content —
896 loaded fresh inside this method's write transaction — by appending a
897 ``[[title]]`` wiki-link, rather than being passed in precomputed by the
898 caller. This keeps the read-modify-write atomic and closes the
899 lost-update window where a concurrent save committed after the caller's
900 precondition read would otherwise be clobbered. Mutually exclusive with
901 ``content``.
902 """
903 _validate_title(title)
904 # Empty title is allowed on create's route-level guard, but here on
905 # update an explicit empty string would erase a previously valid
906 # title with no semantic meaning. Reject it.
907 if title is not None and not title:
908 raise ValueError("title must not be empty")
909 if content is not None:
910 _assert_content_size(content)
911 _validate_tags(tags)
912 try:
913 with get_user_db_session(self.username) as session:
914 document = self._get_note_in_session(session, note_id)
915 if document is None:
916 return False
918 # Append-mode (accept_suggested_link): derive the new body from
919 # the FRESHLY loaded row inside this write transaction, rather
920 # than from a value the caller read in an earlier, now-closed
921 # session. Doing the read-modify-write in one transaction closes
922 # the lost-update window where a concurrent save committed
923 # between the caller's precondition read and this write would
924 # otherwise be silently clobbered.
925 _append_reparse_only = False
926 if _append_link_title is not None:
927 current = document.text_content or ""
928 # If the exact wiki-link text is already in the body, don't
929 # DUPLICATE the line — but STILL reparse links so the target
930 # NoteLink is created/updated. This covers two cases:
931 # (a) a concurrent second accept of the SAME suggestion
932 # (double-submit / two tabs / a retry after a
933 # processed-but-timed-out response) whose first accept
934 # already appended the line and created the NoteLink —
935 # the reparse is then an idempotent no-op; and
936 # (b) accepting a suggestion for a link the user had TYPED
937 # while it was still unresolved (no NoteLink row exists
938 # yet, e.g. the target note was created later). Skipping
939 # the reparse here would make the user's explicit accept
940 # a silent no-op that never creates the link.
941 if f"[[{_append_link_title}]]" in current:
942 content = None
943 _append_reparse_only = True
944 else:
945 content = (
946 current.rstrip() + f"\n\n[[{_append_link_title}]]\n"
947 )
948 _assert_content_size(content)
950 content_changed = (
951 content is not None and content != document.text_content
952 )
953 title_changed = title is not None and title != document.title
954 # Tags are part of the version state (they participate in
955 # _compute_version_hash). A tag-only edit must produce a
956 # snapshot — otherwise the hash improvement is dead code on
957 # this path and the user's version history silently misses
958 # tag-only changes.
959 #
960 # Use sorted() so reordering the same tag set
961 # (e.g. ["b","a"] vs ["a","b"]) doesn't flip the trigger.
962 # _compute_version_hash already sorts before hashing, so an
963 # order-only "change" would just create a dedup-absorbed
964 # snapshot trigger, wasting work; aligning the comparison
965 # keeps trigger and hash consistent.
966 tags_changed = tags is not None and sorted(
967 t for t in tags if isinstance(t, str)
968 ) != sorted(
969 t for t in (document.tags or []) if isinstance(t, str)
970 )
971 old_content = document.text_content
973 if title is not None:
974 document.title = title
975 if content is not None:
976 document.text_content = content
977 document.character_count = len(content)
978 document.word_count = len(content.split())
979 document.document_hash = self._compute_content_hash(
980 f"{note_id}:{content}"
981 )
982 document.file_size = len(content.encode("utf-8"))
983 if tags is not None:
984 document.tags = tags
985 if favorite is not None:
986 document.favorite = favorite
988 # Atomicity: document update + version snapshot + link
989 # rewrite all share the same transaction. Pre-fix, each
990 # ran in its own session, so a crash between them left a
991 # committed content change with no MANUAL_SAVE row
992 # or with zero outgoing links. The AI change-summary
993 # generation still runs off-thread — it's enqueued AFTER
994 # commit so a failed transaction won't leave a stranded
995 # background task.
996 version_id: Optional[str] = None
997 version_created = False
998 # The version-snapshot INSERT is flushed inside
999 # _create_version_snapshot_in_session, so a concurrent
1000 # identical-content save's UNIQUE(uix_note_version_content)
1001 # collision can fire at THAT flush, not only at the commit
1002 # below. Wrap the whole snapshot + link-reparse + commit block
1003 # so it is caught and recovered in one place — previously the
1004 # recovery guarded only commit(), so a flush-time collision
1005 # 500'd a benign concurrent save (the recovery was dead code).
1006 try:
1007 if content_changed or title_changed or tags_changed:
1008 (
1009 version_id,
1010 version_created,
1011 ) = self._create_version_snapshot_in_session(
1012 session,
1013 note_id=note_id,
1014 title=document.title,
1015 content=document.text_content,
1016 tags=document.tags,
1017 change_type=NoteChangeType.MANUAL_SAVE.value,
1018 change_summary=None,
1019 )
1020 self._prune_versions_in_session(session, note_id)
1022 if content_changed or _append_reparse_only:
1023 # In append-reparse-only mode `content` is None (the body
1024 # already held the link text); reparse the CURRENT body
1025 # so the override still binds the NoteLink to the target.
1026 self._parse_and_update_links_in_session(
1027 session,
1028 note_id,
1029 content
1030 if content is not None
1031 else (document.text_content or ""),
1032 link_overrides=_link_overrides,
1033 )
1035 # Stale-vector fix: when the note's content or title
1036 # changes, mark its embeddings stale (both indexed-state
1037 # sources) so the auto-index worker re-embeds AND the RAG
1038 # status report doesn't keep showing it as indexed. See
1039 # _mark_note_stale_for_reindex_in_session.
1040 if content_changed or title_changed:
1041 self._mark_note_stale_for_reindex_in_session(
1042 session, note_id
1043 )
1045 session.commit()
1046 except IntegrityError as exc:
1047 # Concurrency: two same-note saves carrying byte-identical
1048 # content both pass the in-session dedup pre-check in
1049 # _create_version_snapshot_in_session (neither sees the
1050 # other's uncommitted row), then race to INSERT the same
1051 # (document_id, content_hash). The winner commits; the
1052 # loser hits UNIQUE(uix_note_version_content) — at the
1053 # snapshot flush above or at the commit. That
1054 # collision means the identical version already exists and
1055 # the winner already persisted the identical document
1056 # content — so the user's save IS reflected. Recover
1057 # (rollback + return success) instead of 500'ing.
1058 #
1059 # Discriminator: only the version-dedup constraint carries
1060 # "content_hash" (SQLite reports the offending columns,
1061 # Postgres the constraint name uix_note_version_content —
1062 # both contain "content_hash"). Any other IntegrityError
1063 # (e.g. a NoteLink uix_note_link violation from the link
1064 # reparse) must still surface, so re-raise it.
1065 self._rollback_quietly(session)
1066 err = (
1067 str(exc.orig).lower() if exc.orig else str(exc).lower()
1068 )
1069 if "content_hash" not in err:
1070 raise
1071 logger.debug(
1072 "Concurrent identical-content save of note {} "
1073 "collided on the version-dedup UNIQUE; the existing "
1074 "version already represents this content. Treating as "
1075 "success.",
1076 note_id,
1077 )
1078 # The winner persisted the identical VERSIONED state
1079 # (title/content/tags — the fields in the version
1080 # hash), but ``favorite`` is not hashed: a loser that
1081 # also carried a pin change had it rolled back with
1082 # the rest of its transaction, so returning success
1083 # here would silently drop the pin. Re-apply it in a
1084 # fresh transaction — a favorite-only UPDATE cannot
1085 # collide on the version constraint. A failure here
1086 # propagates (outer handler) rather than reporting a
1087 # success the DB doesn't reflect.
1088 if favorite is not None:
1089 refetched = self._get_note_in_session(session, note_id)
1090 if ( 1090 ↛ 1096line 1090 didn't jump to line 1096 because the condition on line 1090 was always true
1091 refetched is not None
1092 and refetched.favorite != favorite
1093 ):
1094 refetched.favorite = favorite
1095 session.commit()
1096 return True
1098 logger.info("Updated note {}", note_id)
1100 # Post-commit: enqueue the LLM change-summary worker. The
1101 # snapshot row already exists with change_summary=None; the
1102 # worker UPDATEs it when the LLM returns. Only schedule it
1103 # for a genuinely NEW row: when the dedup check absorbed
1104 # the snapshot into an existing version (e.g. the user
1105 # reverted to an exact prior state), version_id points at a
1106 # HISTORICAL row whose change_summary the worker would
1107 # overwrite with a description of this much later edit.
1108 if (
1109 version_id
1110 and version_created
1111 and content_changed
1112 and content is not None
1113 ):
1114 # NB: intentionally NOT gated on `old_content` being
1115 # truthy. A first-content edit of a previously blank/NULL
1116 # note (old_content "" or None) is still worth summarizing
1117 # — summarize_changes handles it cheaply (`if not
1118 # old_content: return "Note created"`, no LLM call). Gating
1119 # on truthy old_content left that MANUAL_SAVE version's
1120 # change_summary NULL forever.
1121 # Capture the DB password from the request thread now;
1122 # the worker runs on a stdlib ThreadPoolExecutor which
1123 # does NOT copy ContextVar state, so the worker cannot
1124 # locate the password itself.
1125 dbpw = _capture_request_db_password(self.username)
1126 if dbpw is None:
1127 # Without the password the worker can't open the
1128 # encrypted DB to write the summary. Pre-fix this
1129 # was silently dropped by the worker itself; log
1130 # explicitly and skip the submit so operators can
1131 # see WHY summaries are missing.
1132 logger.warning(
1133 "Skipping AI change-summary for version {}: "
1134 "DB password not available in request thread.",
1135 version_id,
1136 )
1137 else:
1138 _submit_summary_task(
1139 _populate_change_summary_async,
1140 self.username,
1141 dbpw,
1142 version_id,
1143 old_content,
1144 content,
1145 )
1147 return True
1149 except Exception:
1150 # ``get_user_db_session``'s context manager does NOT rollback
1151 # on exception (it returns a cached request- or thread-scoped
1152 # session whose lifecycle is managed elsewhere). Helpers like
1153 # ``_create_version_snapshot_in_session`` call ``session.flush()``
1154 # at the end so a downstream failure (e.g. in link reparse)
1155 # would otherwise leave the document UPDATE durably flushed
1156 # without ever committing properly. Rollback explicitly here
1157 # so the atomicity guarantee holds.
1158 # ``session`` is bound iff we entered the with block; locals().get
1159 # avoids NameError when the failure happens before then, and
1160 # _rollback_quietly no-ops on None.
1161 self._rollback_quietly(locals().get("session"))
1162 logger.exception("Error updating note {}", note_id)
1163 raise
1165 def delete_note(self, note_id: str) -> bool:
1166 """Delete a note.
1168 Before the row is removed we collect the set of collections this note
1169 was indexed into, because the ``Document → DocumentCollection``
1170 cascade fires on delete and would otherwise hide them. After the
1171 delete commits we purge the corresponding ``DocumentChunk`` rows for
1172 each indexed collection via
1173 ``LibraryRAGService.purge_document_chunks`` — NOT
1174 ``remove_document_from_rag``, which looks the document up by its
1175 (now cascade-deleted) ``DocumentCollection`` join row and would
1176 silently no-op.
1178 Both halves of the RAG state are purged: the FAISS vectors first
1179 (``purge_document_vectors`` — replace-on-reindex can never fire
1180 again for a deleted document id, and the shared search flows
1181 rehydrate snippets straight from the ``DocumentChunk`` rows in the
1182 (encrypted) DB by id, with no Document-existence check, so
1183 lingering vectors would serve the deleted note's text
1184 indefinitely), then the ``DocumentChunk`` rows
1185 (``purge_document_chunks``). The ghost-hit filter in
1186 ``NoteAIService.semantic_search`` / ``find_similar_passages``
1187 remains as the safety net for cleanup failures and pre-fix
1188 leftovers.
1189 """
1190 try:
1191 indexed_collection_ids: List[str] = []
1192 with get_user_db_session(self.username) as session:
1193 document = self._get_note_in_session(session, note_id)
1194 if document is None:
1195 return False
1197 # Capture (before the cascade destroys them) every collection
1198 # where this note might have vectors to purge: the UNION of
1199 # DocumentCollection.indexed=True AND collections that actually
1200 # have chunk rows. The chunk-row half is essential — an edited
1201 # note has indexed=False during the edit->reindex window while
1202 # its pre-edit chunks/vectors still exist, so gating on the
1203 # indexed flag alone would strand orphaned vectors.
1204 indexed_rows = (
1205 session.query(DocumentCollection.collection_id)
1206 .filter_by(document_id=note_id, indexed=True)
1207 .all()
1208 )
1209 chunk_rows = (
1210 session.query(DocumentChunk.collection_name)
1211 .filter_by(source_type="document", source_id=note_id)
1212 .distinct()
1213 .all()
1214 )
1215 indexed_collection_ids = list(
1216 {row[0] for row in indexed_rows}
1217 | {
1218 row[0].removeprefix("collection_")
1219 for row in chunk_rows
1220 if row[0]
1221 }
1222 )
1224 session.delete(document)
1225 session.commit()
1226 logger.info("Deleted note {}", note_id)
1228 # Best-effort FAISS / chunk cleanup. Failures must not surface
1229 # to the user (the DB delete already committed) but should be
1230 # logged for ops visibility.
1231 if indexed_collection_ids:
1232 dbpw = _capture_request_db_password(self.username)
1233 from ...services.rag_service_factory import get_rag_service
1235 for collection_id in indexed_collection_ids:
1236 try:
1237 # A FRESH service per collection, built via the
1238 # factory so it resolves THAT collection's stored
1239 # embedding model/provider (get_rag_service reads
1240 # Collection settings; a hardcoded-default
1241 # LibraryRAGService would compute the wrong index
1242 # hash and silently purge nothing for any
1243 # non-default-embedding collection). Per-collection
1244 # instances also avoid one collection's cached
1245 # ``rag_index_record`` / loaded vector index
1246 # carrying over into another collection's
1247 # ``purge_document_vectors`` call.
1248 with get_rag_service(
1249 self.username,
1250 collection_id=collection_id,
1251 db_password=dbpw,
1252 ) as rag:
1253 # Vectors BEFORE chunk rows: purge_document_vectors
1254 # -> VectorIndex.delete looks up the DocumentChunk
1255 # rows by (source_type, source_id, collection_name)
1256 # to learn which FAISS int ids to remove, and
1257 # deletes those matching rows itself. The rows
1258 # must still exist when this runs, or the lookup
1259 # finds nothing and the vectors (and their
1260 # persisted text) are stranded in the FAISS store
1261 # forever. purge_document_chunks below then only
1262 # has to catch whatever purge_document_vectors
1263 # didn't match (e.g. no current FAISS index).
1264 rag.purge_document_vectors(note_id, collection_id)
1265 # purge_document_chunks, not
1266 # remove_document_from_rag: the Document (and
1267 # its DocumentCollection join row) is already
1268 # cascade-deleted, so the join-row lookup in
1269 # remove_document_from_rag would no-op and
1270 # leave the chunks orphaned.
1271 rag.purge_document_chunks(note_id, collection_id)
1272 except Exception:
1273 # exception=True keeps the stack trace. `dbpw` is a
1274 # live local, but frame-local values render only under
1275 # the opt-in LDR_LOGURU_DIAGNOSE dev flag — every
1276 # persistent sink sets diagnose=False — so the password
1277 # cannot leak into logs in normal operation.
1278 logger.opt(exception=True).error(
1279 "Failed to remove RAG entries for note {} "
1280 "from collection {}",
1281 note_id,
1282 collection_id,
1283 )
1285 # Guaranteed backstop: delete any DocumentChunk rows the per-
1286 # collection purge above could not reach (get_rag_service failing
1287 # because the request-thread password is unavailable, a transient
1288 # FAISS/DB error, a stored-config mismatch). chunk_text holds the
1289 # note's content and collection search rehydrates hits purely by
1290 # DocumentChunk.id — a surviving row would keep serving the deleted
1291 # note's text. The matching orphaned vector then resolves to no row
1292 # and is filtered out. Mirrors DocumentDeletionService's backstop.
1293 try:
1294 with get_user_db_session(self.username) as sweep_session:
1295 sweep_session.query(DocumentChunk).filter_by(
1296 source_type="document", source_id=note_id
1297 ).delete(synchronize_session=False)
1298 sweep_session.commit()
1299 except Exception:
1300 logger.opt(exception=True).error(
1301 "Failed to sweep residual chunk rows for note {}", note_id
1302 )
1304 return True
1306 except Exception:
1307 logger.exception("Error deleting note {}", note_id)
1308 raise
1310 def _build_filtered_notes_query(
1311 self,
1312 session: Session,
1313 collection_id: Optional[str],
1314 search: Optional[str],
1315 pinned_only: bool,
1316 ):
1317 """Build the filtered Document query used by both ``list_notes``
1318 and ``count_notes``. Centralising the filter shape keeps the
1319 list response and its ``total`` count in lockstep.
1320 """
1321 source_type_id = self._get_note_source_type_id(session)
1322 query = session.query(Document).filter(
1323 Document.source_type_id == source_type_id
1324 )
1326 if collection_id: 1326 ↛ 1327line 1326 didn't jump to line 1327 because the condition on line 1326 was never true
1327 query = query.join(DocumentCollection).filter(
1328 DocumentCollection.collection_id == collection_id
1329 )
1331 if search:
1332 search_pattern = f"%{self._escape_like(search)}%"
1333 query = query.filter(
1334 or_(
1335 Document.title.ilike(search_pattern, escape="\\"),
1336 Document.text_content.ilike(search_pattern, escape="\\"),
1337 )
1338 )
1340 if pinned_only:
1341 query = query.filter(Document.favorite.is_(True))
1343 return query
1345 def count_notes(
1346 self,
1347 collection_id: Optional[str] = None,
1348 search: Optional[str] = None,
1349 pinned_only: bool = False,
1350 ) -> int:
1351 """Count notes matching the same filter as ``list_notes``.
1353 Used by the list endpoint so the frontend can render real
1354 pagination controls (page-N-of-M, "load more" with bounded
1355 progress, etc.) without having to call ``list_notes`` with a
1356 huge limit just to compute the size.
1357 """
1358 with get_user_db_session(self.username) as session:
1359 return self._build_filtered_notes_query(
1360 session, collection_id, search, pinned_only
1361 ).count()
1363 def list_notes(
1364 self,
1365 collection_id: Optional[str] = None,
1366 search: Optional[str] = None,
1367 pinned_only: bool = False,
1368 limit: int = 100,
1369 offset: int = 0,
1370 ) -> List[Dict[str, Any]]:
1371 """List notes with optional filtering."""
1372 try:
1373 with get_user_db_session(self.username) as session:
1374 query = self._build_filtered_notes_query(
1375 session, collection_id, search, pinned_only
1376 )
1378 query = query.order_by(
1379 Document.favorite.desc(),
1380 Document.updated_at.desc(),
1381 # Stable tiebreaker: without it, notes sharing an
1382 # updated_at (bulk-created) have no guaranteed order, so
1383 # offset pagination ("Load more") could repeat or skip
1384 # rows between pages; mirrors the mention/link panels.
1385 Document.id.desc(),
1386 )
1388 query = query.offset(offset).limit(limit)
1389 documents = query.all()
1391 if not documents:
1392 return []
1394 doc_ids = [d.id for d in documents]
1396 # Batched aggregates: replaces 3 per-row sub-queries inside
1397 # _document_to_note_dict with 3 grouped queries total. Default
1398 # limit=100 went from 300 extra queries/page to 3.
1399 collection_counts = dict(
1400 session.query(
1401 DocumentCollection.document_id,
1402 func.count(DocumentCollection.id),
1403 )
1404 .filter(DocumentCollection.document_id.in_(doc_ids))
1405 .group_by(DocumentCollection.document_id)
1406 .all()
1407 )
1408 research_counts = dict(
1409 session.query(
1410 NoteResearch.document_id,
1411 func.count(NoteResearch.id),
1412 )
1413 .filter(NoteResearch.document_id.in_(doc_ids))
1414 .group_by(NoteResearch.document_id)
1415 .all()
1416 )
1417 indexed_rows = (
1418 session.query(DocumentCollection)
1419 .filter(
1420 DocumentCollection.document_id.in_(doc_ids),
1421 DocumentCollection.indexed.is_(True),
1422 )
1423 .all()
1424 )
1425 indexed_by_doc: Dict[str, DocumentCollection] = {}
1426 for row in indexed_rows: 1426 ↛ 1428line 1426 didn't jump to line 1428 because the loop on line 1426 never started
1427 # Mirror prior .first() semantics: first indexed row wins
1428 indexed_by_doc.setdefault(row.document_id, row)
1430 return [
1431 self._document_to_note_dict(
1432 doc,
1433 session,
1434 _collection_count=collection_counts.get(doc.id, 0),
1435 _research_count=research_counts.get(doc.id, 0),
1436 _indexed_coll=indexed_by_doc.get(doc.id),
1437 include_full_content=False,
1438 )
1439 for doc in documents
1440 ]
1442 except Exception:
1443 logger.exception("Error listing notes")
1444 raise
1446 def add_to_collection(self, note_id: str, collection_id: str) -> bool:
1447 """Add a note to a collection."""
1448 try:
1449 with get_user_db_session(self.username) as session:
1450 # Only operate on notes — every other mutating method guards
1451 # with _is_note, but these collection helpers did not, so a
1452 # non-note Document id could get note-collection rows attached.
1453 doc = self._get_note_in_session(session, note_id)
1454 if doc is None:
1455 return False
1457 # Validate the target collection exists before inserting the
1458 # join row. Otherwise a stale/bogus collection_id trips the FK
1459 # constraint and surfaces as a generic 500 instead of a clean
1460 # 404 (mirrors create_note, which filters to valid ids).
1461 collection = (
1462 session.query(Collection)
1463 .filter_by(id=collection_id)
1464 .first()
1465 )
1466 if collection is None:
1467 raise LookupError( # noqa: TRY301 — except logs and re-raises to the route as a 404
1468 f"Collection {collection_id} not found"
1469 )
1471 existing = (
1472 session.query(DocumentCollection)
1473 .filter_by(document_id=note_id, collection_id=collection_id)
1474 .first()
1475 )
1477 if existing: 1477 ↛ 1480line 1477 didn't jump to line 1480 because the condition on line 1477 was always true
1478 return False
1480 doc_coll = DocumentCollection(
1481 document_id=note_id,
1482 collection_id=collection_id,
1483 indexed=False,
1484 )
1485 session.add(doc_coll)
1486 session.commit()
1487 logger.info(
1488 f"Added note {note_id} to collection {collection_id}"
1489 )
1490 return True
1492 except IntegrityError:
1493 # TOCTOU on UNIQUE(document_id, collection_id): a concurrent add of
1494 # the same note+collection raced past the existing-row check above.
1495 # get_user_db_session already rolled the shared session back; return
1496 # the idempotent "already a member" outcome (route → 409) instead of
1497 # letting the constraint violation surface as a 500.
1498 return False
1499 except Exception:
1500 logger.exception("Error adding note {} to collection", note_id)
1501 raise
1503 def remove_from_collection(self, note_id: str, collection_id: str) -> bool:
1504 """Remove a note from a collection.
1506 Raises ValueError when ``collection_id`` is the system Notes
1507 collection: it is the persistent home of every note (see
1508 ``_get_or_create_notes_collection``), and the deletion services
1509 rely on that invariant — a note unlinked from Notes but still in
1510 a user collection would be hard-deleted as an "orphan" when that
1511 collection is deleted.
1512 """
1513 try:
1514 was_indexed = False
1515 with get_user_db_session(self.username) as session:
1516 doc = self._get_note_in_session(session, note_id)
1517 if doc is None:
1518 return False
1520 collection = (
1521 session.query(Collection)
1522 .filter_by(id=collection_id)
1523 .first()
1524 )
1525 if (
1526 collection is not None
1527 and collection.collection_type == "notes"
1528 ):
1529 raise ValueError( # noqa: TRY301 — needs the db session; except logs and re-raises
1530 "Cannot remove a note from the Notes collection — "
1531 "it is the permanent home of every note"
1532 )
1534 doc_coll = (
1535 session.query(DocumentCollection)
1536 .filter_by(document_id=note_id, collection_id=collection_id)
1537 .first()
1538 )
1540 link_existed = doc_coll is not None
1541 if doc_coll: 1541 ↛ 1572line 1541 didn't jump to line 1572 because the condition on line 1541 was always true
1542 # Purge if this collection is flagged indexed OR the note
1543 # still has chunk rows here — the chunk-row half catches the
1544 # edit->reindex window where indexed=False but the pre-edit
1545 # chunks/vectors still exist. Captured before the link is
1546 # deleted.
1547 was_indexed = bool(doc_coll.indexed) or (
1548 session.query(DocumentChunk.id)
1549 .filter_by(
1550 source_type="document",
1551 source_id=note_id,
1552 collection_name=f"collection_{collection_id}",
1553 )
1554 .first()
1555 is not None
1556 )
1557 session.delete(doc_coll)
1558 session.commit()
1559 logger.info(
1560 f"Removed note {note_id} from collection {collection_id}"
1561 )
1562 else:
1563 # The link is already gone (a concurrent removal). Do NOT
1564 # return early: a racing index_document could have (re)created
1565 # chunk rows for this (note, collection) AFTER the link was
1566 # deleted, and there is no other route-level retry for that
1567 # pair — an early return would leave the removed note's
1568 # plaintext searchable here forever. Fall through to the
1569 # unconditional backstop chunk sweep below (which removes the
1570 # plaintext rows); the full RAG vector purge is skipped since
1571 # there is no link to unlink.
1572 was_indexed = False
1574 # Best-effort FAISS + chunk cleanup for the collection the note was
1575 # unlinked from — mirrors delete_note. Without it, the note's chunk
1576 # rows and vectors linger in that collection's index, so the removed
1577 # note stays fully searchable there. Vectors BEFORE chunk rows (the
1578 # int-id lookup needs the rows to still exist). Uses get_rag_service
1579 # so THAT collection's stored embedding model resolves the right
1580 # index hash. Failures are logged, not surfaced (the unlink already
1581 # committed).
1582 if was_indexed:
1583 dbpw = _capture_request_db_password(self.username)
1584 from ...services.rag_service_factory import get_rag_service
1586 try:
1587 with get_rag_service(
1588 self.username,
1589 collection_id=collection_id,
1590 db_password=dbpw,
1591 ) as rag:
1592 rag.purge_document_vectors(note_id, collection_id)
1593 rag.purge_document_chunks(note_id, collection_id)
1594 except Exception:
1595 # exception=True keeps the stack trace. `dbpw` (the DB
1596 # password) is a live local, but frame-local values render
1597 # only under the opt-in LDR_LOGURU_DIAGNOSE dev flag — every
1598 # persistent sink sets diagnose=False — so it cannot leak.
1599 logger.opt(exception=True).error(
1600 "Failed to purge RAG entries for note {} removed from "
1601 "collection {}",
1602 note_id,
1603 collection_id,
1604 )
1606 # Guaranteed backstop, scoped to THIS collection only (the note
1607 # survives in its other collections): delete any DocumentChunk rows
1608 # the purge above could not reach, so the unlinked note's text stops
1609 # surfacing in this collection's search even when the RAG purge
1610 # failed. Mirrors DocumentDeletionService's scoped backstop.
1611 try:
1612 with get_user_db_session(self.username) as sweep_session:
1613 sweep_session.query(DocumentChunk).filter_by(
1614 source_type="document",
1615 source_id=note_id,
1616 collection_name=f"collection_{collection_id}",
1617 ).delete(synchronize_session=False)
1618 sweep_session.commit()
1619 except Exception:
1620 logger.opt(exception=True).error(
1621 "Failed to sweep residual chunk rows for note {} in "
1622 "collection {}",
1623 note_id,
1624 collection_id,
1625 )
1626 # False when the link was already gone (nothing to unlink), True when
1627 # this call performed the removal — preserves the prior contract.
1628 return link_existed
1630 except Exception:
1631 logger.exception("Error removing note {} from collection", note_id)
1632 raise
1634 def get_note_collections(self, note_id: str) -> List[Dict[str, Any]]:
1635 """Get collections a note belongs to."""
1636 try:
1637 with get_user_db_session(self.username) as session:
1638 # Use JOIN to avoid N+1 query issue
1639 results = (
1640 session.query(Collection, DocumentCollection)
1641 .join(
1642 DocumentCollection,
1643 Collection.id == DocumentCollection.collection_id,
1644 )
1645 .filter(DocumentCollection.document_id == note_id)
1646 .all()
1647 )
1649 return [
1650 {
1651 "id": coll.id,
1652 "name": coll.name,
1653 # The UI hides the remove control for the system
1654 # Notes collection (every note's persistent home;
1655 # remove_from_collection refuses it server-side).
1656 "collection_type": coll.collection_type,
1657 "indexed": dc.indexed,
1658 "chunk_count": dc.chunk_count or 0,
1659 }
1660 for coll, dc in results
1661 ]
1663 except Exception:
1664 logger.exception("Error getting collections for note {}", note_id)
1665 raise
1667 def get_note_research(self, note_id: str) -> List[Dict[str, Any]]:
1668 """Get research runs triggered from a note."""
1669 try:
1670 with get_user_db_session(self.username) as session:
1671 note_research = (
1672 session.query(NoteResearch)
1673 .options(joinedload(NoteResearch.research))
1674 .filter_by(document_id=note_id)
1675 .order_by(NoteResearch.display_order.asc())
1676 .all()
1677 )
1679 return [
1680 {
1681 "id": nr.id,
1682 "research_id": nr.research_id,
1683 "search_engine": (nr.research.research_meta or {})
1684 .get("submission", {})
1685 .get("search_engine")
1686 if nr.research
1687 else None,
1688 "research_mode": nr.research.mode
1689 if nr.research
1690 else None,
1691 "query_used": nr.research.query
1692 if nr.research
1693 else None,
1694 "display_order": nr.display_order,
1695 "is_collapsed": nr.is_collapsed,
1696 "created_at": nr.created_at.isoformat()
1697 if nr.created_at
1698 else None,
1699 }
1700 for nr in note_research
1701 ]
1703 except Exception:
1704 logger.exception("Error getting research for note {}", note_id)
1705 raise
1707 # Max retries on display_order UNIQUE collision. The old value of 5
1708 # was contention-fragile: colliding threads re-read MAX(display_order)
1709 # in lockstep, so under 6+ concurrent same-note links one thread could
1710 # lose all 5 races and 500 the user (dropping the link). Each retry can
1711 # only fail because another writer landed at the same display_order, and
1712 # the number of concurrent NoteResearch inserts is itself bounded by
1713 # MAX_RESEARCH_PER_NOTE — so a budget comfortably above that, combined
1714 # with the randomized jitter below, makes losing every race effectively
1715 # impossible while never masking a non-retryable error.
1716 _LINK_RESEARCH_MAX_RETRIES = MAX_RESEARCH_PER_NOTE + 10
1718 def link_research_to_note(
1719 self,
1720 note_id: str,
1721 research_id: str,
1722 ) -> int:
1723 """Link a research run to a note.
1725 Concurrency: two same-user requests (e.g. double-click or two
1726 tabs) can both observe the same MAX(display_order) and try to
1727 insert with the same display_order. The UNIQUE constraint on
1728 (document_id, display_order) added in migration 0021 turns the
1729 race into an IntegrityError; we catch it, rollback, recompute
1730 MAX, and retry up to a small bound. The
1731 (document_id, research_id) UniqueConstraint is separate — that
1732 one signals "research already linked", which surfaces as
1733 IntegrityError too. We distinguish the two by looking for the
1734 ``display_order`` token in the error: SQLite reports the offending
1735 COLUMNS (not the constraint name) and Postgres reports the constraint
1736 name ``uix_note_research_display_order`` — both contain
1737 ``display_order``, while the duplicate-link and FK errors do not. Only
1738 the display_order collision is retryable; everything else surfaces
1739 immediately so a legitimate duplicate link (or an FK violation) isn't
1740 masked by burning retries.
1741 """
1742 for attempt in range(self._LINK_RESEARCH_MAX_RETRIES): 1742 ↛ 1843line 1742 didn't jump to line 1843 because the loop on line 1742 didn't complete
1743 session = None
1744 try:
1745 with get_user_db_session(self.username) as session:
1746 # Every other mutating method guards with _is_note;
1747 # without this, a NoteResearch row can be attached to
1748 # any Document (e.g. an uploaded PDF) — orphaned rows
1749 # the notes UI never renders.
1750 doc = self._get_note_in_session(session, note_id)
1751 if doc is None:
1752 raise ValueError( # noqa: TRY301 — needs the db session; except rolls back and re-raises
1753 f"Note {note_id} not found"
1754 )
1756 # Enforce MAX_RESEARCH_PER_NOTE where rows are created
1757 # (this is the only NoteResearch creation site). The
1758 # cap previously existed only in reorder_note_research,
1759 # so a note could grow past it and then every reorder
1760 # of the (necessarily full) list failed on the same
1761 # cap — permanently un-reorderable.
1762 existing_count = (
1763 session.query(func.count(NoteResearch.id))
1764 .filter_by(document_id=note_id)
1765 .scalar()
1766 )
1767 if existing_count >= MAX_RESEARCH_PER_NOTE:
1768 raise ValueError( # noqa: TRY301 — needs the db session; except rolls back and re-raises
1769 f"too many linked researches "
1770 f"(max {MAX_RESEARCH_PER_NOTE})"
1771 )
1773 # Use MAX(display_order) + 1, not COUNT, so deleting an
1774 # earlier link can't make the next insert reuse an
1775 # existing display_order. Research runs CASCADE-delete
1776 # NoteResearch rows when a ResearchHistory is removed,
1777 # so COUNT < MAX+1 is a real possibility.
1778 max_order = (
1779 session.query(
1780 func.coalesce(
1781 func.max(NoteResearch.display_order), -1
1782 )
1783 )
1784 .filter_by(document_id=note_id)
1785 .scalar()
1786 )
1788 note_research = NoteResearch(
1789 document_id=note_id,
1790 research_id=research_id,
1791 display_order=max_order + 1,
1792 )
1793 session.add(note_research)
1794 session.commit()
1795 logger.info(
1796 "Linked research {} to note {}", research_id, note_id
1797 )
1798 return note_research.id
1800 except IntegrityError as exc:
1801 # The shared per-user request session is NOT rolled back by the
1802 # context manager on exception (it only closes a session it
1803 # created), so without an explicit rollback the next retry reuses
1804 # a poisoned session and fails identically.
1805 self._rollback_quietly(session)
1806 msg = str(exc.orig).lower() if exc.orig else str(exc).lower()
1807 # Only a (document_id, display_order) collision is retryable.
1808 # The duplicate (document_id, research_id) link and FK violations
1809 # contain no "display_order" token → surface immediately.
1810 if "display_order" not in msg:
1811 raise
1812 if attempt + 1 == self._LINK_RESEARCH_MAX_RETRIES:
1813 logger.exception(
1814 "Giving up linking research {} to note {} after "
1815 "{} display_order collisions",
1816 research_id,
1817 note_id,
1818 self._LINK_RESEARCH_MAX_RETRIES,
1819 )
1820 raise
1821 logger.debug(
1822 "display_order collision linking research {} to note "
1823 "{} (attempt {}); retrying",
1824 research_id,
1825 note_id,
1826 attempt + 1,
1827 )
1828 # Backoff with randomized jitter so colliding threads
1829 # don't re-read MAX(display_order) in perfect lockstep and
1830 # collide again on the very next attempt. Spreading the
1831 # retries breaks the synchronized contention that made a
1832 # fixed, no-backoff budget exhaustible under load.
1833 # noqa S311: jitter for retry-backoff scheduling, not a
1834 # security/crypto context — non-CSPRNG randomness is fine.
1835 time.sleep(random.uniform(0, 0.01 * (attempt + 1))) # noqa: S311
1836 continue
1837 except Exception:
1838 self._rollback_quietly(session)
1839 logger.exception("Error linking research to note {}", note_id)
1840 raise
1842 # Unreachable — the loop either returns or raises.
1843 raise RuntimeError("link_research_to_note exhausted retries")
1845 def get_notes_for_research(
1846 self, research_id: str
1847 ) -> Optional[List[Dict[str, Any]]]:
1848 """List the notes linked to a research run, newest link first.
1850 The reverse direction of ``get_research_for_note`` — powers the
1851 Notes panel on the research results page. Returns ``None`` when
1852 the research run doesn't exist (the route maps that to 404,
1853 distinct from "exists but has no notes yet" → ``[]``).
1855 Column-projected like ``get_backlinks``: ``text_content`` can be
1856 megabytes per note, and this renders a card list that needs only
1857 a short preview.
1858 """
1859 from ....database.models import ResearchHistory
1861 preview_len = self.LIST_CONTENT_PREVIEW_CHARS
1862 with get_user_db_session(self.username) as session:
1863 if (
1864 session.query(ResearchHistory.id)
1865 .filter_by(id=research_id)
1866 .first()
1867 is None
1868 ):
1869 return None
1870 rows = (
1871 session.query(
1872 Document.id,
1873 Document.title,
1874 func.substr(Document.text_content, 1, preview_len + 1),
1875 Document.updated_at,
1876 NoteResearch.created_at,
1877 )
1878 .join(NoteResearch, NoteResearch.document_id == Document.id)
1879 .filter(NoteResearch.research_id == research_id)
1880 .order_by(NoteResearch.created_at.desc())
1881 .all()
1882 )
1883 notes = []
1884 for note_id, title, preview, updated_at, linked_at in rows:
1885 preview = self._truncate_preview(preview, preview_len)
1886 notes.append(
1887 {
1888 "id": note_id,
1889 "title": title,
1890 "content_preview": preview,
1891 "updated_at": updated_at.isoformat()
1892 if updated_at
1893 else None,
1894 "linked_at": linked_at.isoformat()
1895 if linked_at
1896 else None,
1897 }
1898 )
1899 return notes
1901 def _create_note_with_cleanup(
1902 self, title, content, tags, link_fn, context_label
1903 ):
1904 """Create a note and run its post-create ``link_fn`` step
1905 atomically: if linking raises, delete the just-created note so no
1906 orphan survives, then re-raise the original error.
1908 Shared by ``create_note_for_research`` /
1909 ``create_note_for_document`` — the two paths that create-then-link,
1910 where a mid-step failure would otherwise leave a note the user
1911 never asked for. ``context_label`` is the full link-step phrase up
1912 to (and including) its preposition, so the log reads naturally
1913 (e.g. ``"Linking research <id> to"`` → "... to fresh note ...").
1914 """
1915 note_id = self.create_note(title=title, content=content, tags=tags)
1916 try:
1917 link_fn(note_id)
1918 except Exception:
1919 logger.exception(
1920 "{} fresh note {} failed; removing the note so the "
1921 "operation stays all-or-nothing",
1922 context_label,
1923 note_id,
1924 )
1925 try:
1926 self.delete_note(note_id)
1927 except Exception:
1928 # Both halves failed — surface the original linking error;
1929 # the orphan note is visible (not silently lost data).
1930 logger.exception(
1931 "Cleanup of orphan note {} also failed", note_id
1932 )
1933 raise
1934 return note_id
1936 def create_note_for_research(
1937 self,
1938 research_id: str,
1939 title: Optional[str] = None,
1940 content: Optional[str] = None,
1941 tags: Optional[List[str]] = None,
1942 quote: Optional[str] = None,
1943 prefix: Optional[str] = None,
1944 suffix: Optional[str] = None,
1945 ) -> str:
1946 """Create a note and link it to a research run, all-or-nothing.
1948 Used by the results page ("Add note", "Save as note", clip). The
1949 research row is checked FIRST so a bogus id gets a clean
1950 ``LookupError`` (route → 404) instead of creating an orphan note
1951 and then FK-failing on the link. If the link step fails anyway
1952 (e.g. a concurrent research delete), the just-created note is
1953 removed so the caller never observes a half-done operation.
1955 ``title``/``content`` default to a starter derived from the
1956 research query, so the "Add note" button needs no user input
1957 before navigating into the editor.
1958 """
1959 from ....database.models import ResearchHistory
1961 with get_user_db_session(self.username) as session:
1962 research = (
1963 session.query(ResearchHistory.id, ResearchHistory.query)
1964 .filter_by(id=research_id)
1965 .first()
1966 )
1967 if research is None:
1968 raise LookupError("Research not found")
1970 query_text = (research.query or "").strip()
1971 if not title:
1972 title = f"Notes on: {query_text}"[:MAX_TITLE_LENGTH]
1973 if not content:
1974 label = escape_markdown_link_label(query_text or research_id)
1975 content = (
1976 f"> Notes on the research run "
1977 f"[{label}](/results/{research_id}).\n\n"
1978 )
1980 def _link(new_note_id):
1981 self.link_research_to_note(new_note_id, research_id)
1982 if quote:
1983 # An inline annotation: also record the quote anchor so the
1984 # results page can highlight the passage (NULL-quote
1985 # references are plain links and skip this).
1986 self._add_reference(
1987 new_note_id,
1988 target_research_id=research_id,
1989 quote=quote,
1990 prefix=prefix,
1991 suffix=suffix,
1992 )
1994 return self._create_note_with_cleanup(
1995 title, content, tags, _link, f"Linking research {research_id} to"
1996 )
1998 def _add_reference(
1999 self,
2000 note_id: str,
2001 target_document_id: Optional[str] = None,
2002 target_research_id: Optional[str] = None,
2003 quote: Optional[str] = None,
2004 prefix: Optional[str] = None,
2005 suffix: Optional[str] = None,
2006 ) -> int:
2007 """Insert a NoteReference row (see the model for semantics)."""
2008 with get_user_db_session(self.username) as session:
2009 ref = NoteReference(
2010 note_id=note_id,
2011 target_document_id=target_document_id,
2012 target_research_id=target_research_id,
2013 quote=quote,
2014 prefix=prefix,
2015 suffix=suffix,
2016 )
2017 session.add(ref)
2018 try:
2019 session.commit()
2020 except Exception:
2021 self._rollback_quietly(session)
2022 raise
2023 return ref.id
2025 def get_notes_for_document(
2026 self, document_id: str
2027 ) -> Optional[List[Dict[str, Any]]]:
2028 """List the notes referencing a library document, newest first.
2030 Returns ``None`` when the document doesn't exist (route → 404).
2031 Column-projected like ``get_notes_for_research``; a note with
2032 several references (e.g. two annotations) appears once.
2033 """
2034 preview_len = self.LIST_CONTENT_PREVIEW_CHARS
2035 with get_user_db_session(self.username) as session:
2036 if ( 2036 ↛ 2040line 2036 didn't jump to line 2040 because the condition on line 2036 was never true
2037 session.query(Document.id).filter_by(id=document_id).first()
2038 is None
2039 ):
2040 return None
2041 rows = (
2042 session.query(
2043 Document.id,
2044 Document.title,
2045 func.substr(Document.text_content, 1, preview_len + 1),
2046 Document.updated_at,
2047 func.max(NoteReference.created_at),
2048 )
2049 .join(NoteReference, NoteReference.note_id == Document.id)
2050 .filter(NoteReference.target_document_id == document_id)
2051 .group_by(Document.id)
2052 .order_by(func.max(NoteReference.created_at).desc())
2053 .all()
2054 )
2055 notes = []
2056 for note_id, title, preview, updated_at, linked_at in rows:
2057 preview = self._truncate_preview(preview, preview_len)
2058 notes.append(
2059 {
2060 "id": note_id,
2061 "title": title,
2062 "content_preview": preview,
2063 "updated_at": updated_at.isoformat()
2064 if updated_at
2065 else None,
2066 "linked_at": linked_at.isoformat()
2067 if linked_at
2068 else None,
2069 }
2070 )
2071 return notes
2073 def create_note_for_document(
2074 self,
2075 document_id: str,
2076 title: Optional[str] = None,
2077 content: Optional[str] = None,
2078 tags: Optional[List[str]] = None,
2079 quote: Optional[str] = None,
2080 prefix: Optional[str] = None,
2081 suffix: Optional[str] = None,
2082 ) -> str:
2083 """Create a note referencing a library document, all-or-nothing.
2085 The document-side twin of ``create_note_for_research``. Targets
2086 must be non-note documents: annotating a NOTE is deliberately
2087 unsupported — notes are mutable, so quote anchors would drift on
2088 every edit (the re-anchoring problem); library documents' extracted
2089 text is immutable, like research reports.
2090 """
2091 with get_user_db_session(self.username) as session:
2092 row = (
2093 session.query(Document.id, Document.title)
2094 .filter_by(id=document_id)
2095 .first()
2096 )
2097 if row is None: 2097 ↛ 2098line 2097 didn't jump to line 2098 because the condition on line 2097 was never true
2098 raise LookupError("Document not found")
2099 note_source_type_id = self._get_note_source_type_id(session)
2100 is_note = (
2101 session.query(Document.id)
2102 .filter_by(id=document_id, source_type_id=note_source_type_id)
2103 .first()
2104 is not None
2105 )
2106 if is_note:
2107 raise ValueError(
2108 "notes cannot be annotated — their content is mutable, "
2109 "so quote anchors would drift on edit; annotate library "
2110 "documents or research reports instead"
2111 )
2112 doc_title = (row.title or "").strip() or document_id
2114 if not title:
2115 title = f"Notes on: {doc_title}"[:MAX_TITLE_LENGTH]
2116 if not content:
2117 label = escape_markdown_link_label(doc_title)
2118 content = (
2119 f"> Notes on the document "
2120 f"[{label}](/library/document/{document_id}).\n\n"
2121 )
2123 def _link(new_note_id):
2124 self._add_reference(
2125 new_note_id,
2126 target_document_id=document_id,
2127 quote=quote,
2128 prefix=prefix,
2129 suffix=suffix,
2130 )
2132 return self._create_note_with_cleanup(
2133 title,
2134 content,
2135 tags,
2136 _link,
2137 f"Referencing document {document_id} from",
2138 )
2140 def get_annotations_for_target(
2141 self,
2142 document_id: Optional[str] = None,
2143 research_id: Optional[str] = None,
2144 ) -> List[Dict[str, Any]]:
2145 """List the anchored references (inline annotations) for a target.
2147 Existence of the target is the caller's concern (the routes check
2148 it for the 404); a valid target with no anchors returns ``[]``.
2149 """
2150 preview_len = self.LIST_CONTENT_PREVIEW_CHARS
2151 with get_user_db_session(self.username) as session:
2152 q = (
2153 session.query(
2154 NoteReference.note_id,
2155 NoteReference.quote,
2156 NoteReference.prefix,
2157 NoteReference.suffix,
2158 NoteReference.created_at,
2159 Document.title,
2160 func.substr(Document.text_content, 1, preview_len + 1),
2161 )
2162 .join(Document, Document.id == NoteReference.note_id)
2163 .filter(NoteReference.quote.isnot(None))
2164 )
2165 if document_id is not None:
2166 q = q.filter(NoteReference.target_document_id == document_id)
2167 else:
2168 q = q.filter(NoteReference.target_research_id == research_id)
2169 annotations = []
2170 for row in q.order_by(NoteReference.created_at.asc()).all():
2171 (
2172 note_id,
2173 quote,
2174 prefix,
2175 suffix,
2176 created_at,
2177 title,
2178 preview,
2179 ) = row
2180 preview = self._truncate_preview(preview, preview_len)
2181 annotations.append(
2182 {
2183 "note_id": note_id,
2184 "quote": quote or "",
2185 "prefix": prefix or "",
2186 "suffix": suffix or "",
2187 "created_at": created_at.isoformat()
2188 if created_at
2189 else None,
2190 "note_title": title,
2191 "comment_preview": preview,
2192 }
2193 )
2194 return annotations
2196 def has_annotation(
2197 self,
2198 note_id: str,
2199 document_id: Optional[str] = None,
2200 research_id: Optional[str] = None,
2201 ) -> bool:
2202 """True when an anchored reference ties note_id to the target
2203 (the DELETE routes' precondition)."""
2204 with get_user_db_session(self.username) as session:
2205 q = session.query(NoteReference.id).filter(
2206 NoteReference.note_id == note_id,
2207 NoteReference.quote.isnot(None),
2208 )
2209 if document_id is not None:
2210 q = q.filter(NoteReference.target_document_id == document_id)
2211 else:
2212 q = q.filter(NoteReference.target_research_id == research_id)
2213 return q.first() is not None
2215 def update_note_research(
2216 self,
2217 note_id: str,
2218 research_id: str,
2219 is_collapsed: Optional[bool] = None,
2220 ) -> bool:
2221 """Update a NoteResearch row. Currently supports toggling is_collapsed.
2223 Returns True if the row was found and updated, False otherwise.
2224 """
2225 try:
2226 with get_user_db_session(self.username) as session:
2227 nr = (
2228 session.query(NoteResearch)
2229 .filter_by(document_id=note_id, research_id=research_id)
2230 .first()
2231 )
2232 if not nr:
2233 return False
2234 if is_collapsed is not None:
2235 nr.is_collapsed = bool(is_collapsed)
2236 session.commit()
2237 return True
2238 except Exception:
2239 logger.exception(
2240 f"Error updating note_research for {note_id}/{research_id}"
2241 )
2242 raise
2244 def reorder_note_research(
2245 self, note_id: str, research_ids: List[str]
2246 ) -> bool:
2247 """Atomically reorder NoteResearch rows for a note so their
2248 display_order matches the position in research_ids.
2250 Validates every id belongs to the note before updating.
2251 Returns True on success.
2253 Two-pass write because of the
2254 ``UniqueConstraint('document_id', 'display_order')`` added in
2255 migration 0021: assigning the final order directly would hit
2256 an intermediate state where two rows share the same
2257 display_order (e.g. swapping idx 0 and 2 would try to put
2258 idx-2 at 0 while idx-0 is still at 0). The first pass moves
2259 every row into the negative display_order range — guaranteed
2260 not to collide with the positive target values — and the
2261 second pass assigns the final positive values.
2262 """
2263 if not isinstance(research_ids, list): 2263 ↛ 2264line 2263 didn't jump to line 2264 because the condition on line 2263 was never true
2264 raise ValueError("research_ids must be a list")
2265 if len(research_ids) > MAX_RESEARCH_PER_NOTE: 2265 ↛ 2266line 2265 didn't jump to line 2266 because the condition on line 2265 was never true
2266 raise ValueError(
2267 f"too many research_ids (max {MAX_RESEARCH_PER_NOTE})"
2268 )
2269 # A duplicate id collapses in a set, so a set-equality check alone would
2270 # accept e.g. ['A','B','A'] for a note linking {A,B} and then assign A
2271 # two positions — silently corrupting the order. Reject duplicates.
2272 if len(research_ids) != len(set(research_ids)):
2273 return False
2274 session = None
2275 try:
2276 with get_user_db_session(self.username) as session:
2277 existing = (
2278 session.query(NoteResearch)
2279 .filter_by(document_id=note_id)
2280 .all()
2281 )
2282 existing_ids = {nr.research_id for nr in existing}
2283 if set(research_ids) != existing_ids:
2284 return False
2286 by_id = {nr.research_id: nr for nr in existing}
2287 # Pass 1: park every row in the negative range so the
2288 # second pass can assign 0..N-1 without colliding with
2289 # the existing positive display_orders.
2290 for idx, rid in enumerate(research_ids):
2291 by_id[rid].display_order = -(idx + 1)
2292 session.flush()
2293 # Pass 2: final positive values.
2294 for idx, rid in enumerate(research_ids):
2295 by_id[rid].display_order = idx
2296 session.commit()
2297 return True
2298 except Exception:
2299 # Roll back so a failed flush/commit doesn't leave the shared
2300 # request session poisoned for the rest of the request.
2301 self._rollback_quietly(session)
2302 logger.exception("Error reordering research for note {}", note_id)
2303 raise
2305 # Sentinel used by _document_to_note_dict to distinguish "caller passed
2306 # None as the precomputed value (i.e. document is not indexed)" from
2307 # "caller did not pass anything; query the DB". Plain None can't carry
2308 # both meanings.
2309 _UNSET = object()
2311 # Upper bound on body size returned in list-view responses. Frontend
2312 # only ever shows the first ~150 chars; shipping the entire ``text_content``
2313 # column meant a 200-note page could download multiple MB. The 300-char
2314 # cap leaves headroom over the UI cap so the truncation isn't visible
2315 # at the cut-off boundary.
2316 LIST_CONTENT_PREVIEW_CHARS = 300
2318 def _document_to_note_dict(
2319 self,
2320 document: Document,
2321 session: Session,
2322 _collection_count: Any = _UNSET,
2323 _research_count: Any = _UNSET,
2324 _indexed_coll: Any = _UNSET,
2325 include_full_content: bool = True,
2326 ) -> Dict[str, Any]:
2327 """Convert a Document to a note dict.
2329 The three ``_``-prefixed kwargs let list_notes pre-batch the
2330 aggregate queries (count of collections / research links / first
2331 indexed-collection row) into 3 grouped SQL statements instead of
2332 3*N per-row sub-queries. Single-doc callers (get_note) leave them
2333 unset and pay the per-row cost.
2334 """
2335 if _collection_count is self._UNSET:
2336 _collection_count = (
2337 session.query(DocumentCollection)
2338 .filter_by(document_id=document.id)
2339 .count()
2340 )
2341 if _research_count is self._UNSET:
2342 _research_count = (
2343 session.query(NoteResearch)
2344 .filter_by(document_id=document.id)
2345 .count()
2346 )
2347 if _indexed_coll is self._UNSET:
2348 _indexed_coll = (
2349 session.query(DocumentCollection)
2350 .filter_by(document_id=document.id, indexed=True)
2351 .first()
2352 )
2354 collection_count = _collection_count
2355 research_count = _research_count
2356 indexed_coll = _indexed_coll
2358 text_content = document.text_content or ""
2359 if include_full_content:
2360 content_value: Optional[str] = text_content
2361 content_preview: Optional[str] = None
2362 else:
2363 content_value = None
2364 content_preview = self._truncate_preview(
2365 text_content, self.LIST_CONTENT_PREVIEW_CHARS
2366 )
2368 return {
2369 "id": document.id,
2370 "title": document.title,
2371 "content": content_value,
2372 "content_preview": content_preview,
2373 "tags": document.tags or [],
2374 "pinned": document.favorite,
2375 "is_indexed": indexed_coll is not None,
2376 "chunk_count": indexed_coll.chunk_count if indexed_coll else 0,
2377 "last_indexed_at": indexed_coll.last_indexed_at.isoformat()
2378 if indexed_coll and indexed_coll.last_indexed_at
2379 else None,
2380 "created_at": document.created_at.isoformat()
2381 if document.created_at
2382 else None,
2383 "updated_at": document.updated_at.isoformat()
2384 if document.updated_at
2385 else None,
2386 "collection_count": collection_count,
2387 "research_count": research_count,
2388 }
2390 # =========================================================================
2391 # Version History Methods
2392 # =========================================================================
2394 def _create_version_snapshot_in_session(
2395 self,
2396 session: Session,
2397 note_id: str,
2398 title: str,
2399 content: str,
2400 tags: Optional[List[str]],
2401 change_type: str,
2402 change_summary: Optional[str] = None,
2403 ) -> "tuple[str, bool]":
2404 """Create a NoteVersion row in an existing session.
2406 Caller is responsible for committing. Used by ``restore_with_bookends``
2407 to keep PRE_RESTORE + content update + RESTORE in a single
2408 transaction so process death between writes can't lose the audit
2409 trail. Dedup is preserved via the (document_id, content_hash)
2410 check; on a hit, returns the existing version id without inserting.
2412 Returns ``(version_id, created)``: ``created`` is False when the
2413 dedup check absorbed the snapshot into an existing row. Callers
2414 that schedule the async change-summary worker MUST check it —
2415 passing a dedup-absorbed (historical) version id to the worker
2416 would overwrite that old row's change_summary with a description
2417 of a much later edit.
2419 Note that ``content_hash`` is computed over title + content + tags
2420 via ``_compute_version_hash`` so title-only or tag-only edits
2421 produce a distinct hash and aren't silently deduped against an
2422 existing snapshot. The column name stays ``content_hash`` for
2423 schema compatibility but its semantics are "version-state hash".
2424 """
2425 # NoteVersion.content/tags are NOT NULL (note.py), but the source
2426 # this snapshots — Document.text_content — IS nullable, and the
2427 # restore bookend path passes document fields through uncoerced.
2428 # Coerce here so the version table is never stricter than the row
2429 # it captures (a NULL would raise IntegrityError on flush). Title
2430 # is already NOT NULL upstream.
2431 content = content or ""
2432 tags = tags or []
2433 # PRE_RESTORE and RESTORE are audit bookends: by definition their
2434 # state hash matches an existing version row (PRE_RESTORE = current
2435 # state, which IS the prior MANUAL_SAVE; RESTORE = the target
2436 # version's state). Dedup + UNIQUE(document_id, content_hash) would
2437 # silently drop the bookend, erasing the "user restored here" audit
2438 # signal. Salt the hash with a fresh UUID for these change types so
2439 # each row is guaranteed to insert.
2440 #
2441 # INVARIANT WARNING for future contributors: for PRE_RESTORE and
2442 # RESTORE rows, ``content_hash`` is NOT reproducible from
2443 # ``(title, content, tags)`` via _compute_version_hash — the salt
2444 # is hash-only and is NOT persisted into the ``tags`` column
2445 # (clean user-facing data). Any integrity checker that walks
2446 # NoteVersion rows and recomputes hashes MUST filter on
2447 # ``change_type NOT IN ('pre_restore', 'restore')`` before
2448 # comparing — otherwise it will see every restore event as
2449 # corrupted. Removing the salt to "fix" the divergence would
2450 # silently reintroduce the audit-row-drop bug this guards
2451 # against; if you find yourself reaching for that, add a
2452 # change_type column to the UNIQUE constraint instead.
2453 is_audit_bookend = change_type in _AUDIT_BOOKEND_TYPES
2454 if is_audit_bookend:
2455 salted_tags = list(tags or []) + [
2456 f"__{change_type}__{uuid.uuid4()}"
2457 ]
2458 content_hash = self._compute_version_hash(
2459 title, content, salted_tags
2460 )
2461 else:
2462 content_hash = self._compute_version_hash(title, content, tags)
2463 existing = (
2464 session.query(NoteVersion)
2465 .filter_by(document_id=note_id, content_hash=content_hash)
2466 .first()
2467 )
2468 if existing:
2469 return (existing.id, False)
2471 version_id = str(uuid.uuid4())
2472 version = NoteVersion(
2473 id=version_id,
2474 document_id=note_id,
2475 title=title,
2476 content=content,
2477 tags=tags or [],
2478 change_type=change_type,
2479 change_summary=change_summary,
2480 content_hash=content_hash,
2481 )
2482 session.add(version)
2483 session.flush()
2484 return (version_id, True)
2486 def _mark_note_stale_for_reindex_in_session(
2487 self, session: Session, note_id: str
2488 ) -> None:
2489 """Mark a note's embeddings stale after a content/title change.
2491 Two indexed-state sources must move together:
2493 * ``DocumentCollection.indexed`` — the legacy flag the auto-index
2494 worker scans (it runs with ``force_reindex=False``, so a stale
2495 ``indexed=True`` makes it short-circuit and keep serving the
2496 pre-edit embedding).
2497 * ``RagDocumentStatus`` — row-existence is the *canonical* "indexed"
2498 marker that the RAG status route and ``get_rag_stats`` read.
2500 ``index_document`` writes BOTH on index; the edit path must clear
2501 BOTH or the status report shows an edited note as still-indexed
2502 until (and unless) a re-index actually runs. Deleting the status
2503 row matches the model's "no row = not indexed" contract — an edited
2504 note genuinely isn't currently indexed; the worker re-creates the
2505 row when it re-embeds. Caller commits.
2506 """
2507 session.query(DocumentCollection).filter_by(document_id=note_id).update(
2508 {DocumentCollection.indexed: False},
2509 synchronize_session=False,
2510 )
2511 session.query(RagDocumentStatus).filter_by(document_id=note_id).delete(
2512 synchronize_session=False
2513 )
2515 def _prune_versions_in_session(
2516 self, session: Session, note_id: str
2517 ) -> None:
2518 """Prune oldest versions over MAX_VERSIONS_PER_NOTE.
2520 Caller is responsible for committing. Used by both ``update_note``
2521 (after its in-session snapshot write) and ``restore_with_bookends``
2522 (which can exceed the cap by 2 in one transaction due to
2523 PRE_RESTORE + RESTORE).
2524 """
2525 total = (
2526 session.query(NoteVersion).filter_by(document_id=note_id).count()
2527 )
2528 excess = total - MAX_VERSIONS_PER_NOTE
2529 if excess > 0:
2530 # Audit bookends (PRE_RESTORE / RESTORE) are excluded from
2531 # the prune pool so the "user restored here" trail survives
2532 # heavy editing. If the non-bookend pool can't absorb the
2533 # excess, the cap is allowed to drift up slightly — keeping
2534 # audit history is more important than the exact ceiling.
2535 oldest = (
2536 session.query(NoteVersion)
2537 .filter_by(document_id=note_id)
2538 .filter(NoteVersion.change_type.notin_(_AUDIT_BOOKEND_TYPES))
2539 .order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc())
2540 .limit(excess)
2541 .all()
2542 )
2543 for row in oldest:
2544 session.delete(row)
2545 session.flush()
2547 # Separately bound the bookend pool. Excluding bookends from the prune
2548 # above keeps the restore trail, but each restore writes two
2549 # un-prunable rows, so without this ceiling repeated restores grow
2550 # note_versions without limit. Keep the most recent MAX_BOOKEND_VERSIONS.
2551 bookend_total = (
2552 session.query(NoteVersion)
2553 .filter_by(document_id=note_id)
2554 .filter(NoteVersion.change_type.in_(_AUDIT_BOOKEND_TYPES))
2555 .count()
2556 )
2557 bookend_excess = bookend_total - MAX_BOOKEND_VERSIONS
2558 if bookend_excess > 0:
2559 oldest_bookends = (
2560 session.query(NoteVersion)
2561 .filter_by(document_id=note_id)
2562 .filter(NoteVersion.change_type.in_(_AUDIT_BOOKEND_TYPES))
2563 .order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc())
2564 .limit(bookend_excess)
2565 .all()
2566 )
2567 for row in oldest_bookends:
2568 session.delete(row)
2569 session.flush()
2571 def restore_with_bookends(
2572 self, note_id: str, version_id: str
2573 ) -> "tuple[bool, Optional[str]]":
2574 """Restore a note to a previous version atomically.
2576 Wraps PRE_RESTORE snapshot + content update + RESTORE snapshot in
2577 a single transaction. Earlier code did three separate sessions —
2578 process death between writes could leave a restored note with
2579 no audit-trail row. With this method, either all three writes
2580 land or none do.
2582 Returns ``(success, error_code)``:
2583 * ``(True, None)`` — restore complete
2584 * ``(False, "note_not_found")`` — no document with that id, or
2585 it isn't a note
2586 * ``(False, "version_not_found")`` — version id doesn't exist
2587 for this note
2588 * ``(False, "internal_error")`` — exception during restore;
2589 check logs
2591 The route layer maps these to HTTP statuses.
2592 """
2593 try:
2594 with get_user_db_session(self.username) as session:
2595 document = self._get_note_in_session(session, note_id)
2596 if document is None:
2597 return (False, "note_not_found")
2599 version = (
2600 session.query(NoteVersion)
2601 .filter_by(document_id=note_id, id=version_id)
2602 .first()
2603 )
2604 if not version:
2605 return (False, "version_not_found")
2607 # Capture for post-session use (link re-parse)
2608 v_title = version.title
2609 v_content = version.content
2610 v_tags = version.tags
2612 # PRE_RESTORE snapshot of the current document state
2613 self._create_version_snapshot_in_session(
2614 session,
2615 note_id,
2616 document.title,
2617 document.text_content,
2618 document.tags,
2619 NoteChangeType.PRE_RESTORE.value,
2620 )
2622 # Apply the restore to the document
2623 document.title = v_title
2624 document.text_content = v_content
2625 document.tags = v_tags
2626 content_str = v_content or ""
2627 document.character_count = len(content_str)
2628 document.word_count = len(content_str.split())
2629 document.document_hash = self._compute_content_hash(
2630 f"{note_id}:{content_str}"
2631 )
2632 document.file_size = len(content_str.encode("utf-8"))
2634 # RESTORE snapshot marking the post-restore state
2635 self._create_version_snapshot_in_session(
2636 session,
2637 note_id,
2638 v_title,
2639 v_content,
2640 v_tags,
2641 NoteChangeType.RESTORE.value,
2642 change_summary=f"Restored to version {version_id[:8]}",
2643 )
2645 # Prune to cap; with bookends the count can exceed the
2646 # cap by 2 in a single transaction.
2647 self._prune_versions_in_session(session, note_id)
2649 # Stale-vector fix (mirrors update_note): a restore changes
2650 # content/title, so mark its embeddings stale across both
2651 # indexed-state sources — the auto-index worker re-embeds
2652 # and the RAG status report stops showing the pre-restore
2653 # state as indexed. See _mark_note_stale_for_reindex_in_session.
2654 self._mark_note_stale_for_reindex_in_session(session, note_id)
2656 # Re-parse wiki-links INSIDE the restore transaction (against
2657 # the freshly-restored content), exactly like update_note —
2658 # NOT in a separate post-commit session. A post-commit reparse
2659 # raced a concurrent update_note: that save wrote correct
2660 # NoteLink rows for its NEW content inside its own transaction,
2661 # then this stale reparse (computed from the restored content)
2662 # reverted them, leaving the link graph (backlinks/outgoing/
2663 # unlinked-mentions) inconsistent with Document.text_content.
2664 # Trade-off: a reparse failure now rolls the whole restore back
2665 # (the same contract update_note already has) instead of
2666 # logging and leaving a committed-but-mis-linked restore — an
2667 # acceptable, consistent change for link-graph correctness.
2668 self._parse_and_update_links_in_session(
2669 session, note_id, v_content or ""
2670 )
2672 session.commit()
2674 return (True, None)
2676 except Exception:
2677 logger.exception(
2678 f"Error restoring version {version_id} for note {note_id}"
2679 )
2680 return (False, "internal_error")
2682 # Note: get_note_versions / get_version / restore_version were removed
2683 # in PR #3277 — the route layer (notes_routes.py) reimplements these
2684 # queries directly because they need fine-grained control over the
2685 # session lifetime (e.g., the restore route's PRE_RESTORE/RESTORE
2686 # bookend snapshots). The service-layer versions had zero callers.
2687 # Re-add them only when a non-route caller (CLI, scheduled task)
2688 # actually needs them.
2690 # =========================================================================
2691 # Wiki Linking Methods
2692 # =========================================================================
2694 # NOTE: ``_parse_and_update_links_in_session`` is used by both
2695 # ``create_note`` and ``update_note`` so the content insert/update +
2696 # link rewrite land in a single transaction (atomicity).
2697 def _parse_and_update_links_in_session(
2698 self,
2699 session: Session,
2700 note_id: str,
2701 content: str,
2702 link_overrides: Optional[Dict[str, str]] = None,
2703 ) -> List[Dict[str, Any]]:
2704 """Parse [[wiki-style links]] and update NoteLink rows in an
2705 existing session.
2707 Caller is responsible for committing. Used by ``update_note`` and
2708 ``create_note`` so the content update + link rewrite land in one
2709 transaction — a failure mid-parse rolls the content change back
2710 with it. The standalone ``_parse_and_update_links`` keeps its own
2711 session and is used by ``restore_with_bookends`` which has
2712 explicitly chosen to reparse outside its restore transaction
2713 (documented at the call site).
2715 ``link_overrides`` (lowercased link text → target_document_id) lets
2716 a caller bind a specific ``[[text]]`` to an exact target id.
2717 ``accept_suggested_link`` passes it so a suggestion accepted for one
2718 of several same-titled notes links to THAT note — not whichever the
2719 title resolver happens to pick.
2721 Resolution priority per link: (1) ``link_overrides``; (2) the id this
2722 exact link text resolved to before (the rename-safety cache, now
2723 PREFERRED over a fresh lookup so an accepted/established link stays
2724 id-stable across reparse instead of being silently retargeted to the
2725 oldest same-titled note); (3) a fresh title resolution for brand-new
2726 links. Every id-based target is verified to still be an existing
2727 note before use.
2728 """
2729 # Parse + dedupe targets (first appearance wins) BEFORE applying
2730 # the cap. The cap exists to bound per-save DB work, and duplicate
2731 # texts resolve once regardless — counting each repetition against
2732 # the cap let 1000 repetitions of one ``[[A]]`` silently evict a
2733 # later distinct ``[[B]]``.
2734 parsed_targets: List[str] = []
2735 seen_texts: set[str] = set()
2736 truncated = 0
2737 for raw_link_text in LINK_PATTERN.findall(content):
2738 if len(parsed_targets) >= MAX_LINKS_PER_NOTE:
2739 truncated += 1
2740 continue
2741 # Canonical target text = the portion before the first pipe
2742 # of the Obsidian/Roam ``[[Target|Display]]`` alias syntax.
2743 # The display half is presentation-only; resolution, the
2744 # rename-safety cache key, the stored ``link_text`` and the
2745 # ``resolved_links`` payload all key off the target so the
2746 # frontend linkMap (built from outgoing_links.link_text in
2747 # note-detail.js) matches what ``processWikiLinks`` looks up
2748 # after it splits on the pipe. Storing the raw aliased text
2749 # here would leave every aliased link unresolvable on the
2750 # client and fall back to a slower title search.
2751 target_text, _sep, _display = raw_link_text.partition("|")
2752 target_text = target_text.strip()
2753 if not target_text:
2754 # Empty/whitespace target ("[[ ]]", "[[|alias]]") is not a
2755 # link — the frontend renders it as a literal. Pre-fix the
2756 # raw text was passed through to the resolver, whose
2757 # empty-prefix startswith('') matched EVERY note and
2758 # persisted a phantom NoteLink to the user's oldest note.
2759 continue
2760 text_lc = target_text.lower()
2761 if text_lc in seen_texts:
2762 continue
2763 seen_texts.add(text_lc)
2764 parsed_targets.append(target_text)
2765 if truncated:
2766 logger.warning(
2767 "note {} has more than {} distinct wiki-links; dropping "
2768 "{} matches past the cap to bound per-save DB work",
2769 note_id,
2770 MAX_LINKS_PER_NOTE,
2771 truncated,
2772 )
2774 # Existing rows serve three purposes: the rename-safety cache
2775 # (lowercased link text → target id, so renaming a target doesn't
2776 # nuke the link on the source's next save), the diff base below,
2777 # and preservation of per-row state (id, created_at, and the
2778 # auto_suggested badge from an accepted AI suggestion) on links
2779 # that survive the reparse. If multiple existing rows share the
2780 # same link_text (shouldn't happen — UniqueConstraint is on
2781 # (source, target), not (source, link_text)), the first wins.
2782 existing_rows = (
2783 session.query(NoteLink).filter_by(source_document_id=note_id).all()
2784 )
2785 existing_link_targets: Dict[str, str] = {}
2786 for row in existing_rows:
2787 key = (row.link_text or "").lower().strip()
2788 existing_link_targets.setdefault(key, row.target_document_id)
2789 rows_by_target: Dict[str, NoteLink] = {
2790 row.target_document_id: row for row in existing_rows
2791 }
2793 note_source_type_id = self._get_note_source_type_id(session)
2794 overrides = {
2795 (k or "").lower().strip(): v
2796 for k, v in (link_overrides or {}).items()
2797 }
2799 # Batch-validate every id-based candidate (priorities 1 and 2) in
2800 # one query instead of a per-link primary-key lookup — so a
2801 # stale/deleted/non-note id never resurrects a link, at O(1)
2802 # round-trips instead of O(links).
2803 candidate_ids: set[str] = set()
2804 for target_text in parsed_targets:
2805 text_lc = target_text.lower()
2806 for cid in (
2807 overrides.get(text_lc),
2808 existing_link_targets.get(text_lc),
2809 ):
2810 if cid and cid != note_id:
2811 candidate_ids.add(cid)
2812 valid_targets: Dict[str, str] = {}
2813 candidate_list = sorted(candidate_ids)
2814 for start in range(0, len(candidate_list), _IN_CLAUSE_CHUNK):
2815 chunk = candidate_list[start : start + _IN_CLAUSE_CHUNK]
2816 valid_targets.update(
2817 session.query(Document.id, Document.title)
2818 .filter(
2819 Document.id.in_(chunk),
2820 Document.source_type_id == note_source_type_id,
2821 )
2822 .all()
2823 )
2825 def _valid_note_target(doc_id):
2826 """Return {id, title} if doc_id is an existing note (and not
2827 the source itself), else None."""
2828 if not doc_id or doc_id == note_id:
2829 return None
2830 title = valid_targets.get(doc_id)
2831 return {"id": doc_id, "title": title} if title is not None else None
2833 # Priority-3 pre-pass: batch the exact-title strategy for texts
2834 # priorities 1-2 don't cover. One IN() query over lower(title)
2835 # replaces a per-link table scan; ordering + setdefault implements
2836 # the oldest-wins tie-break (see _resolve_link_internal). The
2837 # mapping back from DB rows to link texts uses _fold_title_ascii,
2838 # which mirrors the database's lower().
2839 fresh_texts = [
2840 t
2841 for t in parsed_targets
2842 if not _valid_note_target(overrides.get(t.lower()))
2843 and not _valid_note_target(existing_link_targets.get(t.lower()))
2844 ]
2845 exact_by_fold: Dict[str, Dict[str, Any]] = {}
2846 folded_list = sorted({_fold_title_ascii(t) for t in fresh_texts})
2847 for start in range(0, len(folded_list), _IN_CLAUSE_CHUNK):
2848 chunk = folded_list[start : start + _IN_CLAUSE_CHUNK]
2849 rows = (
2850 session.query(Document.id, Document.title)
2851 .filter(
2852 Document.source_type_id == note_source_type_id,
2853 func.lower(Document.title).in_(chunk),
2854 )
2855 .order_by(Document.created_at.asc(), Document.id.asc())
2856 .all()
2857 )
2858 for doc_id, title in rows:
2859 exact_by_fold.setdefault(
2860 _fold_title_ascii(title), {"id": doc_id, "title": title}
2861 )
2863 resolved_links: List[Dict[str, Any]] = []
2864 desired: Dict[str, str] = {} # target id → link text (first wins)
2865 prefix_memo: Dict[str, Optional[Dict[str, Any]]] = {}
2866 for target_text in parsed_targets:
2867 text_lc = target_text.lower()
2868 # Priority 1: an explicit override (accept_suggested_link binds
2869 # the exact intended target id, immune to title collisions).
2870 target = _valid_note_target(overrides.get(text_lc))
2871 # Priority 2: the id this exact link text previously resolved to.
2872 # Preferring it over a fresh title lookup keeps an accepted /
2873 # established link pinned to its note across reparse and
2874 # keeps a link alive when its target was renamed.
2875 if not target:
2876 target = _valid_note_target(existing_link_targets.get(text_lc))
2877 # Priority 3: fresh title resolution — batched exact match,
2878 # then the (rare) per-text prefix fallback.
2879 if not target:
2880 fold = _fold_title_ascii(target_text)
2881 target = exact_by_fold.get(fold)
2882 if not target:
2883 if fold not in prefix_memo: 2883 ↛ 2887line 2883 didn't jump to line 2887 because the condition on line 2883 was always true
2884 prefix_memo[fold] = self._resolve_link_prefix(
2885 session, target_text, note_source_type_id
2886 )
2887 target = prefix_memo[fold]
2888 if (
2889 target
2890 and target["id"] != note_id
2891 and target["id"] not in desired
2892 ):
2893 desired[target["id"]] = target_text
2894 resolved_links.append(
2895 {
2896 "link_text": target_text,
2897 "target_id": target["id"],
2898 "target_title": target["title"],
2899 }
2900 )
2902 # Apply as a diff instead of delete-all + recreate: surviving rows
2903 # keep their id, created_at and auto_suggested flag, and an
2904 # unchanged link set writes zero rows.
2905 for target_id, row in rows_by_target.items():
2906 if target_id not in desired:
2907 session.delete(row)
2908 for target_id, link_text in desired.items():
2909 new_text = link_text[:MAX_LINK_TEXT_LENGTH]
2910 row = rows_by_target.get(target_id)
2911 if row is None:
2912 session.add(
2913 NoteLink(
2914 source_document_id=note_id,
2915 target_document_id=target_id,
2916 link_text=new_text,
2917 auto_suggested=False,
2918 )
2919 )
2920 elif row.link_text != new_text: 2920 ↛ 2921line 2920 didn't jump to line 2921 because the condition on line 2920 was never true
2921 row.link_text = new_text
2923 session.flush()
2924 return resolved_links
2926 def _parse_and_update_links(
2927 self, note_id: str, content: str
2928 ) -> List[Dict[str, Any]]:
2929 """Standalone version: opens its own session and commits.
2931 Kept for callers that intentionally manage link reparse outside
2932 their primary transaction (notably ``restore_with_bookends``, where
2933 a link-resolution failure is non-fatal and the restore must not
2934 roll back).
2935 """
2936 try:
2937 with get_user_db_session(self.username) as session:
2938 resolved = self._parse_and_update_links_in_session(
2939 session, note_id, content
2940 )
2941 session.commit()
2942 logger.debug(
2943 f"Updated links for note {note_id}: {len(resolved)} links"
2944 )
2945 return resolved
2946 except Exception:
2947 # Use logger.exception so the stack trace lands in the log.
2948 # Link parsing failures are non-fatal (callers continue), but
2949 # silently dropping the cause makes recurring failures
2950 # invisible to operators.
2951 logger.exception("Failed to parse links for note {}", note_id)
2952 return []
2954 def _resolve_link_internal(
2955 self, session: Session, link_text: str
2956 ) -> Optional[Dict[str, Any]]:
2957 """Resolve a link to a note document.
2959 Supports the Obsidian/Roam pipe alias syntax
2960 ``[[Title|Display Text]]``: the target is everything before the
2961 first ``|``; ``Display Text`` is rendering-only and ignored at
2962 resolution time. Without this split, a link like
2963 ``[[My Note|see here]]`` would search for a literal note titled
2964 "my note|see here" — which never exists — and the link would
2965 silently fall back to a stale cached target id on every save.
2967 Two notes can have the same title (e.g. duplicate "TODO" notes).
2968 Tie-break on ``created_at`` ascending so the link resolves to the
2969 OLDEST match. Resolving to the newest instead reads as more
2970 recency-intuitive, but it silently re-points every existing
2971 ``[[Title]]`` the moment a new note reuses that title (link drift by
2972 spooky-action-at-a-distance); oldest is stable and deterministic
2973 (without the order_by, ``.first()`` is at the DB's discretion and can
2974 flip between calls). Real per-target disambiguation — capturing the
2975 picked note's id at autocomplete time — is tracked as follow-up work.
2976 """
2977 # Strip pipe-alias before lookup. Use partition rather than
2978 # split so a target containing additional pipes (rare but
2979 # legal in markdown) doesn't get further mangled.
2980 target_text, sep, _display = link_text.partition("|")
2981 if sep:
2982 link_text = target_text
2983 link_text = link_text.strip()
2984 # Degenerate target ("[[ ]]" / "[[|alias]]"): an empty key must
2985 # never reach the lookup strategies — a LIKE '%' prefix pattern
2986 # would "resolve" to EVERY note (deterministically the user's
2987 # oldest one). The frontend treats an empty target as a literal,
2988 # not a link; mirror that here.
2989 if not link_text:
2990 return None
2991 source_type_id = self._get_note_source_type_id(session)
2993 # Strategy 1: Exact title match. Fold BOTH sides in SQL: folding
2994 # the link text in Python instead diverges for non-ASCII, because
2995 # SQLite's lower() folds ASCII only while str.lower() folds full
2996 # Unicode — a byte-identical title like "Über Alles" could then
2997 # never match its own link (lower('Über Alles') = 'Über alles',
2998 # but Python sent 'über alles'). SQL-side folding keeps ASCII
2999 # case-insensitivity and makes exact-typed non-ASCII titles
3000 # resolve. (Case-insensitive matching of the non-ASCII letters
3001 # themselves would need a custom collation on the connection.)
3002 document = (
3003 session.query(Document)
3004 .filter(
3005 Document.source_type_id == source_type_id,
3006 func.lower(Document.title) == func.lower(link_text),
3007 )
3008 .order_by(Document.created_at.asc(), Document.id.asc())
3009 .first()
3010 )
3011 if document:
3012 return {"id": document.id, "title": document.title}
3014 return self._resolve_link_prefix(session, link_text, source_type_id)
3016 def _resolve_link_prefix(
3017 self, session: Session, link_text: str, source_type_id: int
3018 ) -> Optional[Dict[str, Any]]:
3019 """Strategy 2: partial (prefix) title match — but only for targets
3020 long enough to be a meaningful prefix. A 1-2 char target like
3021 ``[[i]]`` would otherwise prefix-match and silently persist a
3022 NoteLink to an unrelated note ("Ideas", "Interview notes", ...);
3023 below the minimum length an exact match (Strategy 1) is the only
3024 acceptable resolution. Reuses the same floor as the
3025 unlinked-mention scan.
3027 ``link_text`` must already be stripped. Both sides are folded in
3028 SQL for the same non-ASCII reason as Strategy 1; the pattern is
3029 LIKE-escaped so a target containing wildcards (``[[%]]``,
3030 ``[[_]]``) matches literally instead of over-matching every note.
3031 """
3032 if len(link_text) < self.MIN_MENTION_TITLE_LEN:
3033 return None
3034 prefix_pattern = self._escape_like(link_text) + "%"
3035 document = (
3036 session.query(Document)
3037 .filter(
3038 Document.source_type_id == source_type_id,
3039 func.lower(Document.title).like(
3040 func.lower(prefix_pattern), escape="\\"
3041 ),
3042 )
3043 .order_by(Document.created_at.asc(), Document.id.asc())
3044 .first()
3045 )
3046 if document:
3047 return {"id": document.id, "title": document.title}
3049 return None
3051 # Number of preview chars sent in backlinks / outgoing-links payloads.
3052 # Used by the column-projection SUBSTR so we don't pull full
3053 # text_content (up to 50 MB per Document row) just to slice 200
3054 # chars in Python.
3055 LINK_CONTENT_PREVIEW_CHARS = 200
3057 @staticmethod
3058 def _truncate_preview(raw: Optional[str], preview_len: int) -> str:
3059 """Trim a preview string to ``preview_len`` chars, appending "..."
3060 when the source was longer.
3062 The link-preview queries project ``func.substr(..., 1, preview_len
3063 + 1)`` from the DB — the +1 is the sentinel: SQLite SUBSTR returns
3064 at most preview_len+1 chars, so getting the full +1 char means the
3065 original text_content was longer than preview_len and the preview
3066 should be ellipsized. (SUBSTR uses 1-based indexing via length.)
3067 """
3068 s = raw or ""
3069 return s[:preview_len] + "..." if len(s) > preview_len else s
3071 def get_backlinks(self, note_id: str) -> List[Dict[str, Any]]:
3072 """Get notes that link to the given note.
3074 Errors propagate to the caller so the route layer can return a
3075 proper 500 via ``handle_api_error`` — silently returning ``[]``
3076 on a DB failure would be indistinguishable from "no backlinks"
3077 and mask production issues.
3079 Uses ``func.substr`` to project only the preview prefix of
3080 ``text_content`` from the DB. Pre-fix the JOIN loaded full
3081 Document rows including the unbounded ``text_content`` blob
3082 for every backlink source — at 20 backlinks averaging 100 kB
3083 each, the DB transferred ~2 MB just to slice 4 kB of response.
3084 """
3085 with get_user_db_session(self.username) as session:
3086 preview_len = self.LINK_CONTENT_PREVIEW_CHARS
3087 results = (
3088 session.query(
3089 Document.id,
3090 Document.title,
3091 Document.created_at,
3092 func.substr(
3093 Document.text_content, 1, preview_len + 1
3094 ).label("preview"),
3095 NoteLink.link_text,
3096 )
3097 .join(NoteLink, Document.id == NoteLink.source_document_id)
3098 .filter(NoteLink.target_document_id == note_id)
3099 # Deterministic order so the backlinks panel doesn't reshuffle
3100 # between loads (DB row order is otherwise unspecified).
3101 .order_by(Document.title.asc(), Document.id.asc())
3102 .all()
3103 )
3105 backlinks = []
3106 for row in results:
3107 preview = self._truncate_preview(row.preview, preview_len)
3108 backlinks.append(
3109 {
3110 "id": row.id,
3111 "title": row.title,
3112 "link_text": row.link_text,
3113 "content_preview": preview,
3114 "created_at": row.created_at.isoformat()
3115 if row.created_at
3116 else None,
3117 }
3118 )
3120 return backlinks
3122 def get_outgoing_links(self, note_id: str) -> List[Dict[str, Any]]:
3123 """Get notes that this note links to (outgoing links).
3125 Errors propagate to the caller — see ``get_backlinks`` rationale.
3126 Uses ``func.substr`` column projection for the same reason.
3127 """
3128 with get_user_db_session(self.username) as session:
3129 preview_len = self.LINK_CONTENT_PREVIEW_CHARS
3130 results = (
3131 session.query(
3132 Document.id,
3133 Document.title,
3134 Document.created_at,
3135 func.substr(
3136 Document.text_content, 1, preview_len + 1
3137 ).label("preview"),
3138 NoteLink.link_text,
3139 NoteLink.auto_suggested,
3140 )
3141 .join(NoteLink, Document.id == NoteLink.target_document_id)
3142 .filter(NoteLink.source_document_id == note_id)
3143 # Deterministic order so the outgoing-links panel is stable
3144 # across loads.
3145 .order_by(Document.title.asc(), Document.id.asc())
3146 .all()
3147 )
3149 outgoing = []
3150 for row in results:
3151 preview = self._truncate_preview(row.preview, preview_len)
3152 outgoing.append(
3153 {
3154 "id": row.id,
3155 # Stable FK to the target note. The frontend
3156 # (processWikiLinks) keys its [[link_text]] -> id map on
3157 # target_id to resolve wiki-links by id rather than by a
3158 # fragile title search; without it that lookup was always
3159 # empty and every link fell back to title matching.
3160 "target_id": row.id,
3161 "title": row.title,
3162 "link_text": row.link_text,
3163 "content_preview": preview,
3164 "created_at": row.created_at.isoformat()
3165 if row.created_at
3166 else None,
3167 # True when the link was created by accepting an
3168 # AI suggestion (vs. typed by hand). Lets the UI
3169 # badge AI-suggested links.
3170 "auto_suggested": bool(row.auto_suggested),
3171 }
3172 )
3174 return outgoing
3176 # Titles shorter than this are too generic to mention-match usefully
3177 # (they'd flag almost every note), so unlinked-mention scanning skips them.
3178 MIN_MENTION_TITLE_LEN = 3
3180 def get_unlinked_mentions(
3181 self, note_id: str, limit: int = 20
3182 ) -> List[Dict[str, Any]]:
3183 """Find other notes whose text mentions this note's title but that
3184 don't yet link to it — Obsidian-style "unlinked mentions".
3186 Pure lexical (case-insensitive substring on the title), so it needs no
3187 embeddings/LLM. Notes that already link here are excluded; very short
3188 titles are skipped to avoid matching everything.
3189 """
3190 with get_user_db_session(self.username) as session:
3191 note_source_type_id = self._get_note_source_type_id(session)
3192 if not note_source_type_id: 3192 ↛ 3193line 3192 didn't jump to line 3193 because the condition on line 3192 was never true
3193 return []
3194 note = (
3195 session.query(Document)
3196 .filter_by(id=note_id, source_type_id=note_source_type_id)
3197 .first()
3198 )
3199 title = (note.title or "").strip() if note else ""
3200 if len(title) < self.MIN_MENTION_TITLE_LEN:
3201 return []
3203 # Notes that already link to this note — exclude them.
3204 linked_source_ids = {
3205 row[0]
3206 for row in session.query(NoteLink.source_document_id)
3207 .filter_by(target_document_id=note_id)
3208 .all()
3209 }
3211 # Escape LIKE wildcards in the title so e.g. a "50%" title can't
3212 # match every note.
3213 escaped = self._escape_like(title)
3214 preview_len = self.LINK_CONTENT_PREVIEW_CHARS
3215 query = session.query(
3216 Document.id,
3217 Document.title,
3218 func.substr(Document.text_content, 1, preview_len + 1).label(
3219 "preview"
3220 ),
3221 ).filter(
3222 Document.source_type_id == note_source_type_id,
3223 Document.id != note_id,
3224 Document.text_content.ilike(f"%{escaped}%", escape="\\"),
3225 )
3226 if linked_source_ids:
3227 # Exclude already-linking notes in SQL. Done in Python
3228 # after a ``limit * 3`` over-fetch, a page whose first
3229 # candidates were mostly already linked silently returned
3230 # fewer than ``limit`` mentions even though more existed beyond
3231 # the over-fetch window.
3232 query = query.filter(
3233 Document.id.notin_(list(linked_source_ids))
3234 )
3235 rows = (
3236 # Deterministic order: with no ORDER BY the limited slice was
3237 # whatever order the DB returned, so repeated calls could
3238 # surface different mentions. Newest-updated first, id as
3239 # a stable tiebreaker; mirrors the other link/list panels.
3240 query.order_by(Document.updated_at.desc(), Document.id)
3241 .limit(limit)
3242 .all()
3243 )
3245 return [
3246 {
3247 "id": row.id,
3248 "title": row.title,
3249 "content_preview": self._truncate_preview(
3250 row.preview, preview_len
3251 ),
3252 }
3253 for row in rows
3254 ]
3256 def accept_suggested_link(
3257 self, note_id: str, target_note_id: str
3258 ) -> Optional[Dict[str, Any]]:
3259 """Accept an AI-suggested link by inserting [[Target Title]] into the
3260 note content and creating a NoteLink marked auto_suggested=True.
3262 Returns the updated note dict on success. Returns None only for a
3263 benign precondition that means "nothing to accept" — the note or
3264 target is missing, they are the same note, either is not a note, or
3265 the target title is only wiki-link syntax. A genuine write failure
3266 (e.g. update_note's content-size ValueError, or a DB error) is RAISED,
3267 not collapsed into the same None, so the route can map it to the right
3268 status instead of a misleading 404.
3269 """
3270 try:
3271 with get_user_db_session(self.username) as session:
3272 source = session.query(Document).filter_by(id=note_id).first()
3273 target = (
3274 session.query(Document).filter_by(id=target_note_id).first()
3275 )
3276 if not source or not target:
3277 return None
3279 if source.id == target.id:
3280 logger.warning("Cannot self-link note {}", note_id)
3281 return None
3283 # Check both exist as notes
3284 source_type_id = self._get_note_source_type_id(session)
3285 if ( 3285 ↛ 3289line 3285 didn't jump to line 3289 because the condition on line 3285 was never true
3286 source.source_type_id != source_type_id
3287 or target.source_type_id != source_type_id
3288 ):
3289 return None
3291 # If the note already links to this target (e.g. a hand-typed
3292 # [[Target]]), don't append a duplicate wiki-link or relabel the
3293 # user's link as AI-suggested — accepting is a no-op.
3294 already_linked = (
3295 session.query(NoteLink)
3296 .filter_by(
3297 source_document_id=note_id,
3298 target_document_id=target_note_id,
3299 )
3300 .first()
3301 is not None
3302 )
3303 if already_linked:
3304 return self.get_note(note_id)
3306 # Strip bracket characters from the target title before
3307 # interpolating into the [[...]] wiki-link syntax.
3308 # A title containing "]]" would otherwise corrupt the
3309 # link tokens (e.g. "foo]] [[bar" → "[[foo]] [[bar]]"
3310 # which the parser would split into two separate links).
3311 # Also drop any "|" alias portion: the parser partitions
3312 # the link text on the FIRST pipe and resolves only the
3313 # pre-pipe part, so a title containing "|" would make the
3314 # override key below ("a|b") never match the parser's
3315 # lookup key ("a") — silently bypassing the exact-id
3316 # binding and falling back to title resolution.
3317 # Also collapse internal whitespace (newlines included):
3318 # LINK_PATTERN's character class excludes newlines, so a
3319 # title containing one would append a "[[a\nb]]" literal the
3320 # parser can't match — no NoteLink created, yet the method
3321 # would report success and each retry would append more dead
3322 # text. `" ".join(split())` folds any whitespace run
3323 # (newlines/tabs/multiple spaces) to a single space.
3324 target_title = " ".join(
3325 (target.title or "")
3326 .replace("[", "")
3327 .replace("]", "")
3328 .partition("|")[0]
3329 .split()
3330 )
3331 # Degenerate titles ("|", "[[", empty) strip down to "".
3332 # Appending "[[]]" would persist a dead literal in the
3333 # note body (the parser skips empty link text), so refuse
3334 # instead of silently writing an unlinkable artifact.
3335 # Return None (the method's failure contract — mirrors
3336 # the not-found and self-link branches above).
3337 if not target_title:
3338 logger.warning(
3339 "Cannot accept link {} -> {}: target title contains "
3340 "only wiki-link syntax characters",
3341 note_id,
3342 target_note_id,
3343 )
3344 return None
3346 # Guard the duplicate-title case: if the note already has a
3347 # link whose text matches this title but points to a
3348 # DIFFERENT note (e.g. a hand-typed [[TODO]] resolved to an
3349 # older same-titled note), appending another [[TODO]] would
3350 # not add a second edge — the parser dedupes link text with
3351 # Python str.lower(), so the diff would RETARGET the single
3352 # 'todo' edge to this note and delete the user's existing
3353 # link to the other one. Bare [[title]] can't express two
3354 # different targets for the same text, so refuse rather than
3355 # silently destroy the existing edge.
3356 norm_title = target_title.lower()
3357 sibling_links = (
3358 session.query(NoteLink)
3359 .filter(
3360 NoteLink.source_document_id == note_id,
3361 NoteLink.target_document_id != target_note_id,
3362 )
3363 .all()
3364 )
3365 if any(
3366 (lnk.link_text or "").lower() == norm_title
3367 for lnk in sibling_links
3368 ):
3369 logger.warning(
3370 "Cannot accept link {} -> {}: the note already links "
3371 "'{}' to a different note; bare [[title]] can't "
3372 "disambiguate same-titled targets",
3373 note_id,
3374 target_note_id,
3375 target_title,
3376 )
3377 return None
3379 # Session A above performed only precondition reads. The body
3380 # append itself is NOT computed here from source.text_content:
3381 # that value would be stale by the time update_note's separate
3382 # write transaction runs, so a save landing in between would be
3383 # lost. update_note re-reads the current body and appends inside
3384 # its own transaction (see _append_link_title) — read and write
3385 # are then atomic.
3387 # update_note handles the body append (from the note's CURRENT
3388 # content), versioning, link re-parsing and hash update.
3389 # Bind the appended [[target_title]] to the EXACT target id via an
3390 # override so a title shared by several notes — or a title we had
3391 # to strip brackets from above — links to THIS note, not whichever
3392 # the title resolver would pick. The override id is
3393 # preferred on this parse, and the resulting link's id is then
3394 # preferred on every later reparse (the rename-safety cache), so
3395 # the binding is durable.
3396 self.update_note(
3397 note_id,
3398 _append_link_title=target_title,
3399 _link_overrides={target_title.lower().strip(): target_note_id},
3400 )
3402 # Flag the newly-created NoteLink as auto_suggested=True so UX can
3403 # distinguish it from user-typed links.
3404 with get_user_db_session(self.username) as session:
3405 link = (
3406 session.query(NoteLink)
3407 .filter_by(
3408 source_document_id=note_id,
3409 target_document_id=target_note_id,
3410 )
3411 .first()
3412 )
3413 if link: 3413 ↛ 3417line 3413 didn't jump to line 3417
3414 link.auto_suggested = True
3415 session.commit()
3417 return self.get_note(note_id)
3419 except Exception:
3420 # Real write failures (update_note's ValueError content cap, a DB
3421 # IntegrityError, etc.) must NOT be swallowed into the same None
3422 # the benign preconditions return — that hid genuine failures
3423 # behind a misleading "Link could not be accepted" 404. Log for
3424 # context and re-raise so the route maps it (ValueError -> 400,
3425 # anything else -> 500).
3426 logger.exception(
3427 "Error accepting suggested link {} on {}",
3428 target_note_id,
3429 note_id,
3430 )
3431 raise
3433 def resolve_link(self, link_text: str) -> Optional[Dict[str, Any]]:
3434 """Resolve a [[link]] text to a note.
3436 Public wrapper around ``_resolve_link_internal``. Caps ``link_text``
3437 at ``MAX_LINK_TEXT_LENGTH`` defensively even though ``LINK_PATTERN``
3438 already bounds it — the public route hands its body in untouched.
3439 """
3440 if isinstance(link_text, str) and len(link_text) > MAX_LINK_TEXT_LENGTH:
3441 link_text = link_text[:MAX_LINK_TEXT_LENGTH]
3442 try:
3443 with get_user_db_session(self.username) as session:
3444 return self._resolve_link_internal(session, link_text)
3445 except Exception:
3446 # A DB/session failure must NOT collapse into the same None a
3447 # genuinely unresolvable link returns — the route maps None to a
3448 # 404 "Note not found", which would present an outage as the note
3449 # not existing. Log for context and re-raise so the route's
3450 # handle_api_error surfaces a 500 (same rationale as
3451 # accept_suggested_link above).
3452 logger.exception("Error resolving link")
3453 raise
3455 def search_notes_for_linking(
3456 self, query: str, exclude_note_id: Optional[str] = None, limit: int = 10
3457 ) -> List[Dict[str, Any]]:
3458 """Search notes for the [[link]] autocomplete feature."""
3459 try:
3460 with get_user_db_session(self.username) as session:
3461 source_type_id = self._get_note_source_type_id(session)
3462 search_pattern = f"%{self._escape_like(query)}%"
3464 query_obj = session.query(Document).filter(
3465 Document.source_type_id == source_type_id,
3466 Document.title.ilike(search_pattern, escape="\\"),
3467 )
3469 if exclude_note_id:
3470 query_obj = query_obj.filter(Document.id != exclude_note_id)
3472 documents = (
3473 query_obj.order_by(Document.updated_at.desc())
3474 .limit(limit)
3475 .all()
3476 )
3478 return [
3479 {
3480 "id": doc.id,
3481 "title": doc.title,
3482 "slug": self._generate_slug(doc.title),
3483 }
3484 for doc in documents
3485 ]
3487 except Exception:
3488 # Returning [] here would turn an infrastructure failure into an
3489 # HTTP 200 with an empty autocomplete — indistinguishable from
3490 # "no matching notes". Log for context and re-raise so the route
3491 # surfaces a 500 (matches semantic_search's fix in
3492 # note_ai_service, which removed this exact swallow as masking
3493 # outages).
3494 logger.exception(
3495 "Error searching notes for linking (query_len={})",
3496 len(query) if query else 0,
3497 )
3498 raise