Coverage for src/local_deep_research/database/models/library.py: 99%

1""" 

2Library and document models - Unified architecture. 

3All documents (research downloads and user uploads) are stored in one table. 

4Collections organize documents, with "Library" as the default collection. 

5""" 

6 

7import enum 

8import warnings 

9from datetime import datetime 

10 

11from sqlalchemy import ( 

12 JSON, 

13 Boolean, 

14 Column, 

15 Date, 

16 Enum, 

17 ForeignKey, 

18 Index, 

19 Integer, 

20 LargeBinary, 

21 String, 

22 Text, 

23 UniqueConstraint, 

24) 

25from sqlalchemy.orm import Mapped, backref, mapped_column, relationship 

26from sqlalchemy_utc import UtcDateTime, utcnow 

27 

28from .base import Base 

29 

30 

31class RAGIndexStatus(enum.Enum): 

32 """Status values for RAG indices.""" 

33 

34 ACTIVE = "active" 

35 REBUILDING = "rebuilding" 

36 DEPRECATED = "deprecated" 

37 

38 

39class DocumentStatus(enum.Enum): 

40 """Status values for document processing and downloads.""" 

41 

42 PENDING = "pending" 

43 PROCESSING = "processing" 

44 COMPLETED = "completed" 

45 FAILED = "failed" 

46 

47 

48class EmbeddingProvider(enum.Enum): 

49 """Embedding model provider types. 

50 

51 OPENAI covers both the OpenAI cloud API and any OpenAI-compatible 

52 endpoint (LM Studio, vLLM, llama.cpp server, etc.) — the underlying 

53 provider class reads ``embeddings.openai.base_url`` to target a local 

54 server when set, falling back to the OpenAI cloud when unset. 

55 """ 

56 

57 SENTENCE_TRANSFORMERS = "sentence_transformers" 

58 OLLAMA = "ollama" 

59 OPENAI = "openai" 

60 

61 

62class ExtractionMethod(str, enum.Enum): 

63 """Methods used to extract text from documents.""" 

64 

65 PDF_EXTRACTION = "pdf_extraction" 

66 NATIVE_API = "native_api" 

67 UNKNOWN = "unknown" 

68 

69 

70class ExtractionSource(str, enum.Enum): 

71 """Sources used for text extraction.""" 

72 

73 ARXIV_API = "arxiv_api" 

74 PUBMED_API = "pubmed_api" 

75 PDFPLUMBER = "pdfplumber" 

76 PDFPLUMBER_FALLBACK = "pdfplumber_fallback" 

77 LOCAL_PDF = "local_pdf" 

78 LEGACY_FILE = "legacy_file" 

79 

80 

81class ExtractionQuality(str, enum.Enum): 

82 """Quality levels for extracted text.""" 

83 

84 HIGH = "high" 

85 MEDIUM = "medium" 

86 LOW = "low" 

87 

88 

89class DistanceMetric(str, enum.Enum): 

90 """Distance metrics for vector similarity search.""" 

91 

92 COSINE = "cosine" 

93 L2 = "l2" 

94 DOT_PRODUCT = "dot_product" 

95 

96 

97class IndexType(str, enum.Enum): 

98 """FAISS index types for RAG.""" 

99 

100 FLAT = "flat" 

101 HNSW = "hnsw" 

102 IVF = "ivf" 

103 

104 

105class SplitterType(str, enum.Enum): 

106 """Text splitter types for chunking.""" 

107 

108 RECURSIVE = "recursive" 

109 SEMANTIC = "semantic" 

110 TOKEN = "token" 

111 SENTENCE = "sentence" 

112 

113 

114class PDFStorageMode(str, enum.Enum): 

115 """Storage modes for PDF files.""" 

116 

117 NONE = "none" # Don't store PDFs, text-only 

118 FILESYSTEM = "filesystem" # Store PDFs unencrypted on filesystem 

119 DATABASE = "database" # Store PDFs encrypted in database 

120 

121 

122class SourceType(Base): 

123 """ 

124 Document source types (research_download, user_upload, manual_entry, etc.). 

125 Normalized table for consistent categorization. 

126 """ 

127 

128 __tablename__ = "source_types" 

129 

130 id = Column(String(36), primary_key=True) # UUID 

131 name = Column(String(50), nullable=False, unique=True, index=True) 

132 display_name = Column(String(100), nullable=False) 

133 description = Column(Text) 

134 icon = Column(String(50)) # Icon name for UI 

135 

136 # Timestamps 

137 created_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

138 

139 def __repr__(self): 

140 return ( 

141 f"<SourceType(name='{self.name}', display='{self.display_name}')>" 

142 ) 

143 

144 

145class UploadBatch(Base): 

146 """ 

147 DEPRECATED but DO NOT DELETE. See "Why we can't drop this" below. 

148 

149 A multi-round dead-code audit in 2026-05 confirmed this table is 

150 dormant: no code path creates ``UploadBatch`` rows or sets 

151 ``Document.upload_batch_id`` (declared below). Wiring it up would 

152 have needed a product decision on what defines a "batch" (per 

153 upload submit, per UI session, etc.) and surfacing the grouping in 

154 the upload routes / UI — neither happened. Constructing an 

155 ``UploadBatch`` emits a ``DeprecationWarning`` so future 

156 contributors don't accidentally start writing to it. 

157 

158 Why we can't drop this (and the table) 

159 -------------------------------------- 

160 The same audit attempted to write the table-drop migration and 

161 hit an irreconcilable conflict in this codebase's migration 

162 framework: 

163 

164 * ``0001_initial_schema.py`` builds the initial schema from 

165 ``Base.metadata.create_all()``, so the ORM defines what 

166 "schema at every revision" means. 

167 * ``test_migrations_produce_schema_matching_models`` and 

168 ``test_check_schema_shows_no_missing_tables`` enforce that 

169 the migrated DB matches ``Base.metadata`` exactly. 

170 * ``test_stairway_up_down_up_per_revision`` and 

171 ``test_downgrade_leaves_no_residual_tables`` require that 

172 downgrade restores the schema that existed before the upgrade. 

173 

174 The combination means you cannot drop the table from the DB 

175 without also removing it from the ORM, AND you cannot remove it 

176 from the ORM without breaking either the round-trip tests 

177 (because 0001 stops producing the table at pre-0010 revisions) or 

178 the parity tests (model and migrated schema diverge). 

179 

180 A real removal needs a precursor refactor that converts 

181 ``0001_initial_schema.py`` to an explicit table list before the 

182 drop migration can be added. The benefit of finishing the drop is 

183 roughly 3 KB of empty-table overhead per user DB and one fewer 

184 ORM class. The cost is real migration risk against deployed 

185 encrypted DBs. The audit concluded the trade-off was not worth 

186 it; the deprecation here is the entire shipped fix. 

187 

188 If you still want to delete this, bring fresh evidence that the 

189 cost/benefit has changed and read PR #4178 plus the abandoned 

190 "PR 11b" thread that established this constraint. 

191 """ 

192 

193 __tablename__ = "upload_batches" 

194 

195 id = Column(String(36), primary_key=True) # UUID 

196 collection_id = Column( 

197 String(36), 

198 ForeignKey("collections.id", ondelete="CASCADE"), 

199 nullable=False, 

200 index=True, 

201 ) 

202 uploaded_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

203 file_count = Column(Integer, default=0) 

204 total_size = Column(Integer, default=0) # Total bytes 

205 

206 # Relationships 

207 collection = relationship("Collection", backref="upload_batches") 

208 documents = relationship("Document", backref="upload_batch") 

209 

210 def __init__(self, *args, **kwargs): 

211 warnings.warn( 

212 "UploadBatch is deprecated and scheduled for removal in a " 

213 "follow-up release; the table is dormant and has no production " 

214 "writers.", 

215 DeprecationWarning, 

216 stacklevel=2, 

217 ) 

218 super().__init__(*args, **kwargs) 

219 

220 def __repr__(self): 

221 return f"<UploadBatch(id='{self.id}', files={self.file_count}, size={self.total_size})>" 

222 

223 

224class Document(Base): 

225 """ 

226 Unified document table for all documents (research downloads + user uploads). 

227 """ 

228 

229 __tablename__ = "documents" 

230 

231 id = Column(String(36), primary_key=True) # UUID as string 

232 

233 # Source type (research_download, user_upload, etc.) 

234 source_type_id = Column( 

235 String(36), 

236 ForeignKey("source_types.id"), 

237 nullable=False, 

238 index=True, 

239 ) 

240 

241 # Link to original research resource (for research downloads) - nullable for uploads 

242 resource_id = Column( 

243 Integer, 

244 ForeignKey("research_resources.id", ondelete="SET NULL"), 

245 nullable=True, 

246 index=True, 

247 ) 

248 

249 # Link to research (for research downloads) - nullable for uploads 

250 research_id = Column( 

251 String(36), 

252 ForeignKey("research_history.id", ondelete="CASCADE"), 

253 nullable=True, 

254 index=True, 

255 ) 

256 

257 # Link to upload batch (for user uploads) - nullable for research downloads 

258 upload_batch_id = Column( 

259 String(36), 

260 ForeignKey("upload_batches.id", ondelete="SET NULL"), 

261 nullable=True, 

262 index=True, 

263 ) 

264 

265 # Document identification 

266 document_hash = Column( 

267 String(64), nullable=False, unique=True, index=True 

268 ) # SHA256 for deduplication 

269 original_url = Column(Text, nullable=True) # Source URL (for downloads) 

270 filename = Column(String(500), nullable=True) # Display name (for uploads) 

271 original_filename = Column( 

272 String(500), nullable=True 

273 ) # Original upload name 

274 

275 # File information 

276 file_path = Column( 

277 Text, nullable=True 

278 ) # Path relative to library/uploads root 

279 file_size = Column(Integer, nullable=False) # Size in bytes 

280 file_type = Column(String(50), nullable=False) # pdf, txt, md, html, etc. 

281 mime_type = Column(String(100), nullable=True) # MIME type 

282 

283 # Content storage - text always stored in DB 

284 text_content = Column( 

285 Text, nullable=True 

286 ) # Extracted/uploaded text content 

287 

288 # PDF storage mode (none, filesystem, database) 

289 storage_mode = Column( 

290 String(20), nullable=True, default="database" 

291 ) # PDFStorageMode value 

292 

293 # Metadata 

294 title = Column(Text) # Document title 

295 description = Column(Text) # User description 

296 authors = Column(JSON) # List of authors (for research papers) 

297 published_date = Column(Date, nullable=True) # Publication date 

298 

299 # Academic identifiers (for research papers) 

300 doi = Column(String(255), nullable=True, index=True) 

301 arxiv_id = Column(String(100), nullable=True, index=True) 

302 pmid = Column(String(50), nullable=True, index=True) 

303 pmcid = Column(String(50), nullable=True, index=True) 

304 isbn = Column(String(20), nullable=True) 

305 

306 # Download/Upload information 

307 status = Column( 

308 Enum( 

309 DocumentStatus, values_callable=lambda obj: [e.value for e in obj] 

310 ), 

311 nullable=False, 

312 default=DocumentStatus.COMPLETED, 

313 ) 

314 attempts = Column(Integer, default=1) 

315 error_message = Column(Text, nullable=True) 

316 processed_at = Column(UtcDateTime, nullable=False, default=utcnow()) 

317 last_accessed = Column(UtcDateTime, nullable=True) 

318 

319 # Text extraction metadata (for research downloads from PDFs) 

320 extraction_method = Column( 

321 String(50), nullable=True 

322 ) # pdf_extraction, native_api, etc. 

323 extraction_source = Column( 

324 String(50), nullable=True 

325 ) # arxiv_api, pdfplumber, etc. 

326 extraction_quality = Column(String(20), nullable=True) # high, medium, low 

327 has_formatting_issues = Column(Boolean, default=False) 

328 has_encoding_issues = Column(Boolean, default=False) 

329 character_count = Column(Integer, nullable=True) 

330 word_count = Column(Integer, nullable=True) 

331 

332 # Organization 

333 tags = Column(JSON) # User-defined tags 

334 favorite = Column(Boolean, default=False) 

335 

336 # Timestamps 

337 created_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

338 updated_at = Column( 

339 UtcDateTime, default=utcnow(), onupdate=utcnow(), nullable=False 

340 ) 

341 

342 # Relationships 

343 source_type = relationship("SourceType", backref="documents") 

344 resource = relationship( 

345 "ResearchResource", 

346 foreign_keys="[Document.resource_id]", 

347 backref="documents", 

348 ) 

349 research = relationship("ResearchHistory", backref="documents") 

350 collections = relationship( 

351 "DocumentCollection", 

352 back_populates="document", 

353 cascade="all, delete-orphan", 

354 ) 

355 

356 # Indexes for efficient queries 

357 __table_args__ = ( 

358 Index("idx_source_type", "source_type_id", "status"), 

359 Index("idx_research_documents", "research_id", "status"), 

360 Index("idx_document_type", "file_type", "status"), 

361 Index("idx_document_hash", "document_hash"), 

362 ) 

363 

364 def __repr__(self): 

365 # ID-only repr — earlier versions interpolated the title slice 

366 # which leaks user note content into log lines / SQLAlchemy 

367 # error messages on a multi-tenant deployment. The id is enough 

368 # for debugging; full row data is one query away. 

369 doc_id = self.id[:8] if self.id else "None" 

370 return f"<Document(id={doc_id}, type={self.file_type})>" 

371 

372 

373class DocumentBlob(Base): 

374 """ 

375 Separate table for storing PDF binary content. 

376 SQLite best practices: keep BLOBs in separate table for better query performance. 

377 Stored in encrypted SQLCipher database for security. 

378 """ 

379 

380 __tablename__ = "document_blobs" 

381 

382 # Primary key references Document.id 

383 document_id = Column( 

384 String(36), 

385 ForeignKey("documents.id", ondelete="CASCADE"), 

386 primary_key=True, 

387 nullable=False, 

388 ) 

389 

390 # Binary PDF content 

391 pdf_binary = Column(LargeBinary, nullable=False) 

392 

393 # Hash for integrity verification 

394 blob_hash = Column(String(64), nullable=True, index=True) # SHA256 

395 

396 # Timestamps 

397 stored_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

398 last_accessed = Column(UtcDateTime, nullable=True) 

399 

400 # Relationship 

401 document = relationship( 

402 "Document", 

403 backref=backref("blob", passive_deletes=True), 

404 passive_deletes=True, 

405 ) 

406 

407 def __repr__(self): 

408 size = len(self.pdf_binary) if self.pdf_binary else 0 

409 return f"<DocumentBlob(document_id='{self.document_id[:8]}...', size={size})>" 

410 

411 

412class Collection(Base): 

413 """ 

414 Collections for organizing documents. 

415 'Library' is the default collection for research downloads. 

416 Users can create custom collections for organization. 

417 """ 

418 

419 __tablename__ = "collections" 

420 

421 id: Mapped[str] = mapped_column( 

422 String(36), primary_key=True 

423 ) # UUID as string 

424 name: Mapped[str] = mapped_column(String(255), nullable=False) 

425 description: Mapped[str | None] = mapped_column(Text) 

426 

427 # Collection type (default_library, user_collection, linked_folder) 

428 collection_type: Mapped[str | None] = mapped_column( 

429 String(50), default="user_collection" 

430 ) 

431 

432 # Is this the default library collection? 

433 is_default: Mapped[bool | None] = mapped_column(Boolean, default=False) 

434 

435 # Egress classification: is this collection's content non-sensitive 

436 # ("public") or sensitive ("private")? Defaults to private (False) — the 

437 # safe choice. A private collection is excluded under PUBLIC_ONLY scope 

438 # and forces local LLM/embeddings inference when used, so its chunks 

439 # never reach a cloud model. Operators flip it per-collection. 

440 is_public: Mapped[bool | None] = mapped_column(Boolean, default=False) 

441 

442 # Whether this collection is offered to the research agent (LangGraph) as a 

443 # specialized search tool. Defaults to True (available) so existing 

444 # collections keep their current behaviour; flip it off to declutter the 

445 # agent's tool list when a collection isn't needed for agentic research. 

446 # Independent of is_public / egress scope — this is a usability switch, not 

447 # a security control. 

448 agent_enabled: Mapped[bool | None] = mapped_column(Boolean, default=True) 

449 

450 # A requested full rebuild has not published yet. Kept independently of 

451 # short-lived task history; incremental indexing does not finish a rebuild. 

452 rebuild_pending_since: Mapped[datetime | None] = mapped_column( 

453 UtcDateTime, nullable=True 

454 ) 

455 

456 # Embedding model used for this collection (stored when first indexed) 

457 embedding_model: Mapped[str | None] = mapped_column( 

458 String(100), nullable=True 

459 ) # e.g., 'all-MiniLM-L6-v2', 'nomic-embed-text:latest' 

460 embedding_model_type: Mapped[EmbeddingProvider | None] = mapped_column( 

461 Enum( 

462 EmbeddingProvider, 

463 values_callable=lambda obj: [e.value for e in obj], 

464 ), 

465 nullable=True, 

466 ) 

467 embedding_dimension: Mapped[int | None] = mapped_column( 

468 Integer, nullable=True 

469 ) # Vector dimension 

470 chunk_size: Mapped[int | None] = mapped_column( 

471 Integer, nullable=True 

472 ) # Chunk size used 

473 chunk_overlap: Mapped[int | None] = mapped_column( 

474 Integer, nullable=True 

475 ) # Chunk overlap used 

476 

477 # Advanced embedding configuration options (Issue #1054) 

478 splitter_type: Mapped[str | None] = mapped_column( 

479 String(50), nullable=True 

480 ) # Splitter type: 'recursive', 'semantic', 'token', 'sentence' 

481 text_separators: Mapped[list[str] | None] = mapped_column( 

482 JSON, nullable=True 

483 ) # Text separators for chunking, e.g., ["\n\n", "\n", ". ", " ", ""] 

484 distance_metric: Mapped[str | None] = mapped_column( 

485 String(50), nullable=True 

486 ) # Distance metric: 'cosine', 'l2', 'dot_product' 

487 normalize_vectors: Mapped[bool | None] = mapped_column( 

488 Boolean, nullable=True 

489 ) # Whether to normalize embeddings with L2 

490 index_type: Mapped[str | None] = mapped_column( 

491 String(50), nullable=True 

492 ) # FAISS index type: 'flat', 'hnsw', 'ivf' 

493 

494 # Timestamps 

495 created_at: Mapped[datetime] = mapped_column( 

496 UtcDateTime, default=utcnow(), nullable=False 

497 ) 

498 updated_at: Mapped[datetime] = mapped_column( 

499 UtcDateTime, default=utcnow(), onupdate=utcnow(), nullable=False 

500 ) 

501 

502 # Relationships 

503 document_links: Mapped[list["DocumentCollection"]] = relationship( 

504 "DocumentCollection", 

505 back_populates="collection", 

506 cascade="all, delete-orphan", 

507 ) 

508 linked_folders: Mapped[list["CollectionFolder"]] = relationship( 

509 "CollectionFolder", 

510 back_populates="collection", 

511 cascade="all, delete-orphan", 

512 ) 

513 

514 def __repr__(self): 

515 return f"<Collection(id='{self.id}', name='{self.name}', type='{self.collection_type}')>" 

516 

517 

518class DocumentCollection(Base): 

519 """ 

520 Many-to-many relationship between documents and collections. 

521 Tracks indexing status per collection (documents can be in multiple collections). 

522 """ 

523 

524 __tablename__ = "document_collections" 

525 

526 id: Mapped[int] = mapped_column( 

527 Integer, primary_key=True, autoincrement=True 

528 ) 

529 

530 # Foreign keys 

531 document_id: Mapped[str] = mapped_column( 

532 String(36), 

533 ForeignKey("documents.id", ondelete="CASCADE"), 

534 nullable=False, 

535 index=True, 

536 ) 

537 collection_id: Mapped[str] = mapped_column( 

538 String(36), 

539 ForeignKey("collections.id", ondelete="CASCADE"), 

540 nullable=False, 

541 index=True, 

542 ) 

543 

544 # Indexing status (per collection!) 

545 indexed: Mapped[bool | None] = mapped_column( 

546 Boolean, default=False 

547 ) # Whether indexed for this collection 

548 chunk_count: Mapped[int | None] = mapped_column( 

549 Integer, default=0 

550 ) # Number of chunks in this collection 

551 last_indexed_at: Mapped[datetime | None] = mapped_column( 

552 UtcDateTime, nullable=True 

553 ) 

554 

555 # Timestamps 

556 added_at: Mapped[datetime] = mapped_column( 

557 UtcDateTime, default=utcnow(), nullable=False 

558 ) 

559 

560 # Relationships 

561 document: Mapped["Document"] = relationship( 

562 "Document", back_populates="collections" 

563 ) 

564 collection: Mapped["Collection"] = relationship( 

565 "Collection", back_populates="document_links" 

566 ) 

567 

568 # Ensure one entry per document-collection pair 

569 __table_args__ = ( 

570 UniqueConstraint( 

571 "document_id", "collection_id", name="uix_document_collection" 

572 ), 

573 Index("idx_collection_indexed", "collection_id", "indexed"), 

574 ) 

575 

576 def __repr__(self): 

577 return f"<DocumentCollection(doc_id={self.document_id}, coll_id={self.collection_id}, indexed={self.indexed})>" 

578 

579 

580class DocumentChunk(Base): 

581 """ 

582 Universal chunk storage for RAG across all sources. 

583 Stores text chunks in encrypted database for semantic search. 

584 """ 

585 

586 __tablename__ = "document_chunks" 

587 

588 id: Mapped[int] = mapped_column( 

589 Integer, primary_key=True, autoincrement=True 

590 ) 

591 

592 # Chunk identification 

593 chunk_hash = Column( 

594 String(64), nullable=False, index=True 

595 ) # SHA256 for deduplication 

596 

597 # Source tracking - now points to unified Document table 

598 source_type = Column( 

599 String(20), nullable=False, index=True 

600 ) # 'document', 'folder_file' 

601 source_id = Column( 

602 String(36), nullable=True, index=True 

603 ) # Document.id (UUID as string) 

604 source_path = Column( 

605 Text, nullable=True 

606 ) # File path if local collection source 

607 collection_name = Column( 

608 String(100), nullable=False, index=True 

609 ) # collection_<uuid> 

610 

611 # Chunk content (encrypted in SQLCipher DB) 

612 chunk_text = Column(Text, nullable=False) # The actual chunk text 

613 chunk_index = Column(Integer, nullable=False) # Position in source document 

614 start_char = Column(Integer, nullable=False) # Start character position 

615 end_char = Column(Integer, nullable=False) # End character position 

616 word_count = Column(Integer, nullable=False) # Number of words in chunk 

617 

618 # Embedding metadata 

619 embedding_id = Column( 

620 String(36), nullable=False, unique=True, index=True 

621 ) # UUID for FAISS vector mapping 

622 embedding_model = Column( 

623 String(100), nullable=False 

624 ) # e.g., 'all-MiniLM-L6-v2' 

625 embedding_model_type = Column( 

626 Enum( 

627 EmbeddingProvider, 

628 values_callable=lambda obj: [e.value for e in obj], 

629 ), 

630 nullable=False, 

631 ) 

632 embedding_dimension = Column(Integer, nullable=True) # Vector dimension 

633 

634 # Document metadata (for context) 

635 document_title = Column(Text, nullable=True) # Title of source document 

636 document_metadata = Column( 

637 JSON, nullable=True 

638 ) # Additional metadata from source 

639 

640 # Timestamps 

641 created_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

642 last_accessed = Column(UtcDateTime, nullable=True) 

643 

644 # Indexes for efficient queries. 

645 # NOTE: the old UniqueConstraint("chunk_hash", "collection_name", 

646 # name="uix_chunk_collection") was DROPPED (migration 0023). The RAG store 

647 # is now one-row-per-chunk keyed by the int PK (no cross-document chunk 

648 # sharing), so identical text in two documents legitimately produces two 

649 # rows; the unique constraint would raise IntegrityError on that (and on a 

650 # document with a repeated/blank chunk). chunk_hash stays a plain index for 

651 # query-time result de-duplication. 

652 __table_args__ = ( 

653 Index("idx_chunk_source", "source_type", "source_id"), 

654 Index("idx_chunk_collection", "collection_name", "created_at"), 

655 Index("idx_chunk_embedding", "embedding_id"), 

656 # AUTOINCREMENT so this id is NEVER reused after rows are deleted. 

657 # The id keys the vector store; a plain SQLite ROWID gets recycled, so 

658 # a recycled id could be re-adopted by an unrelated later chunk while a 

659 # stale vector for the old id still lingers in the index — making search 

660 # rehydrate the WRONG document's text. A monotonic id closes that. 

661 {"sqlite_autoincrement": True}, 

662 ) 

663 

664 def __repr__(self): 

665 return f"<DocumentChunk(collection='{self.collection_name}', source_type='{self.source_type}', index={self.chunk_index}, words={self.word_count})>" 

666 

667 

668class DownloadQueue(Base): 

669 """ 

670 Queue for pending document downloads. 

671 Renamed from LibraryDownloadQueue for consistency. 

672 """ 

673 

674 __tablename__ = "download_queue" 

675 

676 id = Column(Integer, primary_key=True, autoincrement=True) 

677 

678 # What to download 

679 resource_id = Column( 

680 Integer, 

681 ForeignKey("research_resources.id", ondelete="CASCADE"), 

682 nullable=False, 

683 unique=True, # One queue entry per resource 

684 ) 

685 research_id = Column(String(36), nullable=False, index=True) 

686 

687 # Target collection (defaults to Library collection) 

688 collection_id = Column( 

689 String(36), 

690 ForeignKey("collections.id", ondelete="SET NULL"), 

691 nullable=True, 

692 index=True, 

693 ) 

694 

695 # Queue management 

696 priority = Column(Integer, default=0) # Higher = more important 

697 status = Column( 

698 Enum( 

699 DocumentStatus, values_callable=lambda obj: [e.value for e in obj] 

700 ), 

701 nullable=False, 

702 default=DocumentStatus.PENDING, 

703 ) 

704 attempts = Column(Integer, default=0) 

705 max_attempts = Column(Integer, default=3) 

706 

707 # Error tracking 

708 last_error = Column(Text, nullable=True) 

709 last_attempt_at = Column(UtcDateTime, nullable=True) 

710 

711 # Timestamps 

712 queued_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

713 completed_at = Column(UtcDateTime, nullable=True) 

714 

715 # Relationships 

716 resource = relationship("ResearchResource", backref="download_queue") 

717 collection = relationship("Collection", backref="download_queue_items") 

718 

719 # Index for the common filter_by(research_id=..., status=...) query 

720 # pattern used by download_bulk, queue_all_undownloaded, and the 

721 # background scheduler. Mirrors Document.idx_research_documents. 

722 __table_args__ = ( 

723 Index("idx_download_queue_research_status", "research_id", "status"), 

724 ) 

725 

726 def __repr__(self): 

727 return f"<DownloadQueue(resource_id={self.resource_id}, status={self.status}, attempts={self.attempts})>" 

728 

729 

730class LibraryStatistics(Base): 

731 """ 

732 Aggregate statistics for the library. 

733 Updated periodically for dashboard display. 

734 """ 

735 

736 __tablename__ = "library_statistics" 

737 

738 id = Column(Integer, primary_key=True, autoincrement=True) 

739 

740 # Document counts 

741 total_documents = Column(Integer, default=0) 

742 total_pdfs = Column(Integer, default=0) 

743 total_html = Column(Integer, default=0) 

744 total_other = Column(Integer, default=0) 

745 

746 # Storage metrics 

747 total_size_bytes = Column(Integer, default=0) 

748 average_document_size = Column(Integer, default=0) 

749 

750 # Research metrics 

751 total_researches_with_downloads = Column(Integer, default=0) 

752 average_documents_per_research = Column(Integer, default=0) 

753 

754 # Download metrics 

755 total_download_attempts = Column(Integer, default=0) 

756 successful_downloads = Column(Integer, default=0) 

757 failed_downloads = Column(Integer, default=0) 

758 pending_downloads = Column(Integer, default=0) 

759 

760 # Academic sources breakdown 

761 arxiv_count = Column(Integer, default=0) 

762 pubmed_count = Column(Integer, default=0) 

763 doi_count = Column(Integer, default=0) 

764 other_count = Column(Integer, default=0) 

765 

766 # Timestamps 

767 calculated_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

768 

769 def __repr__(self): 

770 return f"<LibraryStatistics(documents={self.total_documents}, size={self.total_size_bytes})>" 

771 

772 

773class RAGIndex(Base): 

774 """ 

775 Tracks FAISS indices for RAG collections. 

776 Each full embedding and chunking configuration has its own FAISS index. 

777 """ 

778 

779 __tablename__ = "rag_indices" 

780 

781 id: Mapped[int] = mapped_column( 

782 Integer, primary_key=True, autoincrement=True 

783 ) 

784 

785 # Collection and model identification 

786 collection_name: Mapped[str] = mapped_column( 

787 String(100), nullable=False, index=True 

788 ) # 'collection_<uuid>' 

789 embedding_model: Mapped[str] = mapped_column( 

790 String(100), nullable=False 

791 ) # e.g., 'all-MiniLM-L6-v2' 

792 embedding_model_type: Mapped[EmbeddingProvider] = mapped_column( 

793 Enum( 

794 EmbeddingProvider, 

795 values_callable=lambda obj: [e.value for e in obj], 

796 ), 

797 nullable=False, 

798 ) 

799 embedding_dimension: Mapped[int] = mapped_column( 

800 Integer, nullable=False 

801 ) # Vector dimension 

802 

803 # Index file location 

804 index_path: Mapped[str] = mapped_column( 

805 Text, nullable=False 

806 ) # Path to .faiss file 

807 index_hash: Mapped[str] = mapped_column( 

808 String(64), nullable=False, unique=True, index=True 

809 ) # SHA256 of collection+model for uniqueness 

810 

811 # Chunking parameters used 

812 chunk_size: Mapped[int] = mapped_column(Integer, nullable=False) 

813 chunk_overlap: Mapped[int] = mapped_column(Integer, nullable=False) 

814 

815 # Advanced embedding configuration options (Issue #1054) 

816 splitter_type: Mapped[str | None] = mapped_column( 

817 String(50), nullable=True 

818 ) # Splitter type: 'recursive', 'semantic', 'token', 'sentence' 

819 text_separators: Mapped[list[str] | None] = mapped_column( 

820 JSON, nullable=True 

821 ) # Text separators for chunking, e.g., ["\n\n", "\n", ". ", " ", ""] 

822 distance_metric: Mapped[str | None] = mapped_column( 

823 String(50), nullable=True 

824 ) # Distance metric: 'cosine', 'l2', 'dot_product' 

825 normalize_vectors: Mapped[bool | None] = mapped_column( 

826 Boolean, nullable=True 

827 ) # Whether to normalize embeddings with L2 

828 index_type: Mapped[str | None] = mapped_column( 

829 String(50), nullable=True 

830 ) # FAISS index type: 'flat', 'hnsw', 'ivf' 

831 

832 # Index statistics 

833 chunk_count: Mapped[int | None] = mapped_column( 

834 Integer, default=0 

835 ) # Number of chunks in this index 

836 total_documents: Mapped[int | None] = mapped_column( 

837 Integer, default=0 

838 ) # Number of source documents 

839 

840 # Status 

841 status: Mapped[RAGIndexStatus] = mapped_column( 

842 Enum( 

843 RAGIndexStatus, values_callable=lambda obj: [e.value for e in obj] 

844 ), 

845 nullable=False, 

846 default=RAGIndexStatus.ACTIVE, 

847 ) 

848 is_current: Mapped[bool | None] = mapped_column( 

849 Boolean, default=True 

850 ) # Whether this is the current index for this collection 

851 

852 # Timestamps 

853 created_at: Mapped[datetime] = mapped_column( 

854 UtcDateTime, default=utcnow(), nullable=False 

855 ) 

856 last_updated_at: Mapped[datetime] = mapped_column( 

857 UtcDateTime, default=utcnow(), onupdate=utcnow(), nullable=False 

858 ) 

859 last_used_at: Mapped[datetime | None] = mapped_column( 

860 UtcDateTime, nullable=True 

861 ) # Last time index was searched 

862 

863 __table_args__ = ( 

864 Index("idx_collection_current", "collection_name", "is_current"), 

865 ) 

866 

867 def __repr__(self): 

868 return f"<RAGIndex(collection='{self.collection_name}', model='{self.embedding_model}', chunks={self.chunk_count})>"