Coverage for src/local_deep_research/database/models/library.py: 99%

1""" 

2Library and document models - Unified architecture. 

3All documents (research downloads and user uploads) are stored in one table. 

4Collections organize documents, with "Library" as the default collection. 

5""" 

6 

7import enum 

8import warnings 

9from datetime import datetime 

10 

11from sqlalchemy import ( 

12 JSON, 

13 Boolean, 

14 Column, 

15 Date, 

16 Enum, 

17 ForeignKey, 

18 Index, 

19 Integer, 

20 LargeBinary, 

21 String, 

22 Text, 

23 UniqueConstraint, 

24) 

25from sqlalchemy.orm import Mapped, backref, mapped_column, relationship 

26from sqlalchemy_utc import UtcDateTime, utcnow 

27 

28from .base import Base 

29 

30 

31class RAGIndexStatus(enum.Enum): 

32 """Status values for RAG indices.""" 

33 

34 ACTIVE = "active" 

35 REBUILDING = "rebuilding" 

36 DEPRECATED = "deprecated" 

37 

38 

39class DocumentStatus(enum.Enum): 

40 """Status values for document processing and downloads.""" 

41 

42 PENDING = "pending" 

43 PROCESSING = "processing" 

44 COMPLETED = "completed" 

45 FAILED = "failed" 

46 

47 

48class EmbeddingProvider(enum.Enum): 

49 """Embedding model provider types. 

50 

51 OPENAI covers both the OpenAI cloud API and any OpenAI-compatible 

52 endpoint (LM Studio, vLLM, llama.cpp server, etc.) — the underlying 

53 provider class reads ``embeddings.openai.base_url`` to target a local 

54 server when set, falling back to the OpenAI cloud when unset. 

55 """ 

56 

57 SENTENCE_TRANSFORMERS = "sentence_transformers" 

58 OLLAMA = "ollama" 

59 OPENAI = "openai" 

60 

61 

62class ExtractionMethod(str, enum.Enum): 

63 """Methods used to extract text from documents.""" 

64 

65 PDF_EXTRACTION = "pdf_extraction" 

66 NATIVE_API = "native_api" 

67 UNKNOWN = "unknown" 

68 

69 

70class ExtractionSource(str, enum.Enum): 

71 """Sources used for text extraction.""" 

72 

73 ARXIV_API = "arxiv_api" 

74 PUBMED_API = "pubmed_api" 

75 PDFPLUMBER = "pdfplumber" 

76 PDFPLUMBER_FALLBACK = "pdfplumber_fallback" 

77 LOCAL_PDF = "local_pdf" 

78 LEGACY_FILE = "legacy_file" 

79 

80 

81class ExtractionQuality(str, enum.Enum): 

82 """Quality levels for extracted text.""" 

83 

84 HIGH = "high" 

85 MEDIUM = "medium" 

86 LOW = "low" 

87 

88 

89class DistanceMetric(str, enum.Enum): 

90 """Distance metrics for vector similarity search.""" 

91 

92 COSINE = "cosine" 

93 L2 = "l2" 

94 DOT_PRODUCT = "dot_product" 

95 

96 

97class IndexType(str, enum.Enum): 

98 """FAISS index types for RAG.""" 

99 

100 FLAT = "flat" 

101 HNSW = "hnsw" 

102 IVF = "ivf" 

103 

104 

105class SplitterType(str, enum.Enum): 

106 """Text splitter types for chunking.""" 

107 

108 RECURSIVE = "recursive" 

109 SEMANTIC = "semantic" 

110 TOKEN = "token" 

111 SENTENCE = "sentence" 

112 

113 

114class PDFStorageMode(str, enum.Enum): 

115 """Storage modes for PDF files.""" 

116 

117 NONE = "none" # Don't store PDFs, text-only 

118 FILESYSTEM = "filesystem" # Store PDFs unencrypted on filesystem 

119 DATABASE = "database" # Store PDFs encrypted in database 

120 

121 

122class SourceType(Base): 

123 """ 

124 Document source types (research_download, user_upload, manual_entry, etc.). 

125 Normalized table for consistent categorization. 

126 """ 

127 

128 __tablename__ = "source_types" 

129 

130 id = Column(String(36), primary_key=True) # UUID 

131 name = Column(String(50), nullable=False, unique=True, index=True) 

132 display_name = Column(String(100), nullable=False) 

133 description = Column(Text) 

134 icon = Column(String(50)) # Icon name for UI 

135 

136 # Timestamps 

137 created_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

138 

139 def __repr__(self): 

140 return ( 

141 f"<SourceType(name='{self.name}', display='{self.display_name}')>" 

142 ) 

143 

144 

145class UploadBatch(Base): 

146 """ 

147 DEPRECATED but DO NOT DELETE. See "Why we can't drop this" below. 

148 

149 A multi-round dead-code audit in 2026-05 confirmed this table is 

150 dormant: no code path creates ``UploadBatch`` rows or sets 

151 ``Document.upload_batch_id`` (declared below). Wiring it up would 

152 have needed a product decision on what defines a "batch" (per 

153 upload submit, per UI session, etc.) and surfacing the grouping in 

154 the upload routes / UI — neither happened. Constructing an 

155 ``UploadBatch`` emits a ``DeprecationWarning`` so future 

156 contributors don't accidentally start writing to it. 

157 

158 Why we can't drop this (and the table) 

159 -------------------------------------- 

160 The same audit attempted to write the table-drop migration and 

161 hit an irreconcilable conflict in this codebase's migration 

162 framework: 

163 

164 * ``0001_initial_schema.py`` builds the initial schema from 

165 ``Base.metadata.create_all()``, so the ORM defines what 

166 "schema at every revision" means. 

167 * ``test_migrations_produce_schema_matching_models`` and 

168 ``test_check_schema_shows_no_missing_tables`` enforce that 

169 the migrated DB matches ``Base.metadata`` exactly. 

170 * ``test_stairway_up_down_up_per_revision`` and 

171 ``test_downgrade_leaves_no_residual_tables`` require that 

172 downgrade restores the schema that existed before the upgrade. 

173 

174 The combination means you cannot drop the table from the DB 

175 without also removing it from the ORM, AND you cannot remove it 

176 from the ORM without breaking either the round-trip tests 

177 (because 0001 stops producing the table at pre-0010 revisions) or 

178 the parity tests (model and migrated schema diverge). 

179 

180 A real removal needs a precursor refactor that converts 

181 ``0001_initial_schema.py`` to an explicit table list before the 

182 drop migration can be added. The benefit of finishing the drop is 

183 roughly 3 KB of empty-table overhead per user DB and one fewer 

184 ORM class. The cost is real migration risk against deployed 

185 encrypted DBs. The audit concluded the trade-off was not worth 

186 it; the deprecation here is the entire shipped fix. 

187 

188 If you still want to delete this, bring fresh evidence that the 

189 cost/benefit has changed and read PR #4178 plus the abandoned 

190 "PR 11b" thread that established this constraint. 

191 """ 

192 

193 __tablename__ = "upload_batches" 

194 

195 id = Column(String(36), primary_key=True) # UUID 

196 collection_id = Column( 

197 String(36), 

198 ForeignKey("collections.id", ondelete="CASCADE"), 

199 nullable=False, 

200 index=True, 

201 ) 

202 uploaded_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

203 file_count = Column(Integer, default=0) 

204 total_size = Column(Integer, default=0) # Total bytes 

205 

206 # Relationships 

207 collection = relationship("Collection", backref="upload_batches") 

208 documents = relationship("Document", backref="upload_batch") 

209 

210 def __init__(self, *args, **kwargs): 

211 warnings.warn( 

212 "UploadBatch is deprecated and scheduled for removal in a " 

213 "follow-up release; the table is dormant and has no production " 

214 "writers.", 

215 DeprecationWarning, 

216 stacklevel=2, 

217 ) 

218 super().__init__(*args, **kwargs) 

219 

220 def __repr__(self): 

221 return f"<UploadBatch(id='{self.id}', files={self.file_count}, size={self.total_size})>" 

222 

223 

224class Document(Base): 

225 """ 

226 Unified document table for all documents (research downloads + user uploads). 

227 """ 

228 

229 __tablename__ = "documents" 

230 

231 id = Column(String(36), primary_key=True) # UUID as string 

232 

233 # Source type (research_download, user_upload, etc.) 

234 source_type_id = Column( 

235 String(36), 

236 ForeignKey("source_types.id"), 

237 nullable=False, 

238 index=True, 

239 ) 

240 

241 # Link to original research resource (for research downloads) - nullable for uploads 

242 resource_id = Column( 

243 Integer, 

244 ForeignKey("research_resources.id", ondelete="SET NULL"), 

245 nullable=True, 

246 index=True, 

247 ) 

248 

249 # Link to research (for research downloads) - nullable for uploads 

250 research_id = Column( 

251 String(36), 

252 ForeignKey("research_history.id", ondelete="CASCADE"), 

253 nullable=True, 

254 index=True, 

255 ) 

256 

257 # Link to upload batch (for user uploads) - nullable for research downloads 

258 upload_batch_id = Column( 

259 String(36), 

260 ForeignKey("upload_batches.id", ondelete="SET NULL"), 

261 nullable=True, 

262 index=True, 

263 ) 

264 

265 # Document identification 

266 document_hash = Column( 

267 String(64), nullable=False, unique=True, index=True 

268 ) # SHA256 for deduplication 

269 original_url = Column(Text, nullable=True) # Source URL (for downloads) 

270 filename = Column(String(500), nullable=True) # Display name (for uploads) 

271 original_filename = Column( 

272 String(500), nullable=True 

273 ) # Original upload name 

274 

275 # File information 

276 file_path = Column( 

277 Text, nullable=True 

278 ) # Path relative to library/uploads root 

279 file_size = Column(Integer, nullable=False) # Size in bytes 

280 file_type = Column(String(50), nullable=False) # pdf, txt, md, html, etc. 

281 mime_type = Column(String(100), nullable=True) # MIME type 

282 

283 # Content storage - text always stored in DB 

284 text_content = Column( 

285 Text, nullable=True 

286 ) # Extracted/uploaded text content 

287 

288 # PDF storage mode (none, filesystem, database) 

289 storage_mode = Column( 

290 String(20), nullable=True, default="database" 

291 ) # PDFStorageMode value 

292 

293 # Metadata 

294 title = Column(Text) # Document title 

295 description = Column(Text) # User description 

296 authors = Column(JSON) # List of authors (for research papers) 

297 published_date = Column(Date, nullable=True) # Publication date 

298 

299 # Academic identifiers (for research papers) 

300 doi = Column(String(255), nullable=True, index=True) 

301 arxiv_id = Column(String(100), nullable=True, index=True) 

302 pmid = Column(String(50), nullable=True, index=True) 

303 pmcid = Column(String(50), nullable=True, index=True) 

304 isbn = Column(String(20), nullable=True) 

305 

306 # Download/Upload information 

307 status = Column( 

308 Enum( 

309 DocumentStatus, values_callable=lambda obj: [e.value for e in obj] 

310 ), 

311 nullable=False, 

312 default=DocumentStatus.COMPLETED, 

313 ) 

314 attempts = Column(Integer, default=1) 

315 error_message = Column(Text, nullable=True) 

316 processed_at = Column(UtcDateTime, nullable=False, default=utcnow()) 

317 last_accessed = Column(UtcDateTime, nullable=True) 

318 

319 # Text extraction metadata (for research downloads from PDFs) 

320 extraction_method = Column( 

321 String(50), nullable=True 

322 ) # pdf_extraction, native_api, etc. 

323 extraction_source = Column( 

324 String(50), nullable=True 

325 ) # arxiv_api, pdfplumber, etc. 

326 extraction_quality = Column(String(20), nullable=True) # high, medium, low 

327 has_formatting_issues = Column(Boolean, default=False) 

328 has_encoding_issues = Column(Boolean, default=False) 

329 character_count = Column(Integer, nullable=True) 

330 word_count = Column(Integer, nullable=True) 

331 

332 # Organization 

333 tags = Column(JSON) # User-defined tags 

334 favorite = Column(Boolean, default=False) 

335 

336 # Timestamps 

337 created_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

338 updated_at = Column( 

339 UtcDateTime, default=utcnow(), onupdate=utcnow(), nullable=False 

340 ) 

341 

342 # Relationships 

343 source_type = relationship("SourceType", backref="documents") 

344 resource = relationship( 

345 "ResearchResource", 

346 foreign_keys="[Document.resource_id]", 

347 backref="documents", 

348 ) 

349 research = relationship("ResearchHistory", backref="documents") 

350 collections = relationship( 

351 "DocumentCollection", 

352 back_populates="document", 

353 cascade="all, delete-orphan", 

354 ) 

355 

356 # Indexes for efficient queries 

357 __table_args__ = ( 

358 Index("idx_source_type", "source_type_id", "status"), 

359 Index("idx_research_documents", "research_id", "status"), 

360 Index("idx_document_type", "file_type", "status"), 

361 Index("idx_document_hash", "document_hash"), 

362 ) 

363 

364 def __repr__(self): 

365 # ID-only repr — earlier versions interpolated the title slice 

366 # which leaks user note content into log lines / SQLAlchemy 

367 # error messages on a multi-tenant deployment. The id is enough 

368 # for debugging; full row data is one query away. 

369 doc_id = self.id[:8] if self.id else "None" 

370 return f"<Document(id={doc_id}, type={self.file_type})>" 

371 

372 

373class DocumentBlob(Base): 

374 """ 

375 Separate table for storing PDF binary content. 

376 SQLite best practices: keep BLOBs in separate table for better query performance. 

377 Stored in encrypted SQLCipher database for security. 

378 """ 

379 

380 __tablename__ = "document_blobs" 

381 

382 # Primary key references Document.id 

383 document_id = Column( 

384 String(36), 

385 ForeignKey("documents.id", ondelete="CASCADE"), 

386 primary_key=True, 

387 nullable=False, 

388 ) 

389 

390 # Binary PDF content 

391 pdf_binary = Column(LargeBinary, nullable=False) 

392 

393 # Hash for integrity verification 

394 blob_hash = Column(String(64), nullable=True, index=True) # SHA256 

395 

396 # Timestamps 

397 stored_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

398 last_accessed = Column(UtcDateTime, nullable=True) 

399 

400 # Relationship 

401 document = relationship( 

402 "Document", 

403 backref=backref("blob", passive_deletes=True), 

404 passive_deletes=True, 

405 ) 

406 

407 def __repr__(self): 

408 size = len(self.pdf_binary) if self.pdf_binary else 0 

409 return f"<DocumentBlob(document_id='{self.document_id[:8]}...', size={size})>" 

410 

411 

412class Collection(Base): 

413 """ 

414 Collections for organizing documents. 

415 'Library' is the default collection for research downloads. 

416 Users can create custom collections for organization. 

417 """ 

418 

419 __tablename__ = "collections" 

420 

421 id: Mapped[str] = mapped_column( 

422 String(36), primary_key=True 

423 ) # UUID as string 

424 name: Mapped[str] = mapped_column(String(255), nullable=False) 

425 description: Mapped[str | None] = mapped_column(Text) 

426 

427 # Collection type (default_library, user_collection, linked_folder) 

428 collection_type: Mapped[str | None] = mapped_column( 

429 String(50), default="user_collection" 

430 ) 

431 

432 # Is this the default library collection? 

433 is_default: Mapped[bool | None] = mapped_column(Boolean, default=False) 

434 

435 # Egress classification: is this collection's content non-sensitive 

436 # ("public") or sensitive ("private")? Defaults to private (False) — the 

437 # safe choice. A private collection is excluded under PUBLIC_ONLY scope 

438 # and forces local LLM/embeddings inference when used, so its chunks 

439 # never reach a cloud model. Operators flip it per-collection. 

440 is_public: Mapped[bool | None] = mapped_column(Boolean, default=False) 

441 

442 # Whether this collection is offered to the research agent (LangGraph) as a 

443 # specialized search tool. Defaults to True (available) so existing 

444 # collections keep their current behaviour; flip it off to declutter the 

445 # agent's tool list when a collection isn't needed for agentic research. 

446 # Independent of is_public / egress scope — this is a usability switch, not 

447 # a security control. 

448 agent_enabled: Mapped[bool | None] = mapped_column(Boolean, default=True) 

449 

450 # Embedding model used for this collection (stored when first indexed) 

451 embedding_model: Mapped[str | None] = mapped_column( 

452 String(100), nullable=True 

453 ) # e.g., 'all-MiniLM-L6-v2', 'nomic-embed-text:latest' 

454 embedding_model_type: Mapped[EmbeddingProvider | None] = mapped_column( 

455 Enum( 

456 EmbeddingProvider, 

457 values_callable=lambda obj: [e.value for e in obj], 

458 ), 

459 nullable=True, 

460 ) 

461 embedding_dimension: Mapped[int | None] = mapped_column( 

462 Integer, nullable=True 

463 ) # Vector dimension 

464 chunk_size: Mapped[int | None] = mapped_column( 

465 Integer, nullable=True 

466 ) # Chunk size used 

467 chunk_overlap: Mapped[int | None] = mapped_column( 

468 Integer, nullable=True 

469 ) # Chunk overlap used 

470 

471 # Advanced embedding configuration options (Issue #1054) 

472 splitter_type: Mapped[str | None] = mapped_column( 

473 String(50), nullable=True 

474 ) # Splitter type: 'recursive', 'semantic', 'token', 'sentence' 

475 text_separators: Mapped[list[str] | None] = mapped_column( 

476 JSON, nullable=True 

477 ) # Text separators for chunking, e.g., ["\n\n", "\n", ". ", " ", ""] 

478 distance_metric: Mapped[str | None] = mapped_column( 

479 String(50), nullable=True 

480 ) # Distance metric: 'cosine', 'l2', 'dot_product' 

481 normalize_vectors: Mapped[bool | None] = mapped_column( 

482 Boolean, nullable=True 

483 ) # Whether to normalize embeddings with L2 

484 index_type: Mapped[str | None] = mapped_column( 

485 String(50), nullable=True 

486 ) # FAISS index type: 'flat', 'hnsw', 'ivf' 

487 

488 # Timestamps 

489 created_at: Mapped[datetime] = mapped_column( 

490 UtcDateTime, default=utcnow(), nullable=False 

491 ) 

492 updated_at: Mapped[datetime] = mapped_column( 

493 UtcDateTime, default=utcnow(), onupdate=utcnow(), nullable=False 

494 ) 

495 

496 # Relationships 

497 document_links: Mapped[list["DocumentCollection"]] = relationship( 

498 "DocumentCollection", 

499 back_populates="collection", 

500 cascade="all, delete-orphan", 

501 ) 

502 linked_folders: Mapped[list["CollectionFolder"]] = relationship( 

503 "CollectionFolder", 

504 back_populates="collection", 

505 cascade="all, delete-orphan", 

506 ) 

507 

508 def __repr__(self): 

509 return f"<Collection(id='{self.id}', name='{self.name}', type='{self.collection_type}')>" 

510 

511 

512class DocumentCollection(Base): 

513 """ 

514 Many-to-many relationship between documents and collections. 

515 Tracks indexing status per collection (documents can be in multiple collections). 

516 """ 

517 

518 __tablename__ = "document_collections" 

519 

520 id: Mapped[int] = mapped_column( 

521 Integer, primary_key=True, autoincrement=True 

522 ) 

523 

524 # Foreign keys 

525 document_id: Mapped[str] = mapped_column( 

526 String(36), 

527 ForeignKey("documents.id", ondelete="CASCADE"), 

528 nullable=False, 

529 index=True, 

530 ) 

531 collection_id: Mapped[str] = mapped_column( 

532 String(36), 

533 ForeignKey("collections.id", ondelete="CASCADE"), 

534 nullable=False, 

535 index=True, 

536 ) 

537 

538 # Indexing status (per collection!) 

539 indexed: Mapped[bool | None] = mapped_column( 

540 Boolean, default=False 

541 ) # Whether indexed for this collection 

542 chunk_count: Mapped[int | None] = mapped_column( 

543 Integer, default=0 

544 ) # Number of chunks in this collection 

545 last_indexed_at: Mapped[datetime | None] = mapped_column( 

546 UtcDateTime, nullable=True 

547 ) 

548 

549 # Timestamps 

550 added_at: Mapped[datetime] = mapped_column( 

551 UtcDateTime, default=utcnow(), nullable=False 

552 ) 

553 

554 # Relationships 

555 document: Mapped["Document"] = relationship( 

556 "Document", back_populates="collections" 

557 ) 

558 collection: Mapped["Collection"] = relationship( 

559 "Collection", back_populates="document_links" 

560 ) 

561 

562 # Ensure one entry per document-collection pair 

563 __table_args__ = ( 

564 UniqueConstraint( 

565 "document_id", "collection_id", name="uix_document_collection" 

566 ), 

567 Index("idx_collection_indexed", "collection_id", "indexed"), 

568 ) 

569 

570 def __repr__(self): 

571 return f"<DocumentCollection(doc_id={self.document_id}, coll_id={self.collection_id}, indexed={self.indexed})>" 

572 

573 

574class DocumentChunk(Base): 

575 """ 

576 Universal chunk storage for RAG across all sources. 

577 Stores text chunks in encrypted database for semantic search. 

578 """ 

579 

580 __tablename__ = "document_chunks" 

581 

582 id = Column(Integer, primary_key=True, autoincrement=True) 

583 

584 # Chunk identification 

585 chunk_hash = Column( 

586 String(64), nullable=False, index=True 

587 ) # SHA256 for deduplication 

588 

589 # Source tracking - now points to unified Document table 

590 source_type = Column( 

591 String(20), nullable=False, index=True 

592 ) # 'document', 'folder_file' 

593 source_id = Column( 

594 String(36), nullable=True, index=True 

595 ) # Document.id (UUID as string) 

596 source_path = Column( 

597 Text, nullable=True 

598 ) # File path if local collection source 

599 collection_name = Column( 

600 String(100), nullable=False, index=True 

601 ) # collection_<uuid> 

602 

603 # Chunk content (encrypted in SQLCipher DB) 

604 chunk_text = Column(Text, nullable=False) # The actual chunk text 

605 chunk_index = Column(Integer, nullable=False) # Position in source document 

606 start_char = Column(Integer, nullable=False) # Start character position 

607 end_char = Column(Integer, nullable=False) # End character position 

608 word_count = Column(Integer, nullable=False) # Number of words in chunk 

609 

610 # Embedding metadata 

611 embedding_id = Column( 

612 String(36), nullable=False, unique=True, index=True 

613 ) # UUID for FAISS vector mapping 

614 embedding_model = Column( 

615 String(100), nullable=False 

616 ) # e.g., 'all-MiniLM-L6-v2' 

617 embedding_model_type = Column( 

618 Enum( 

619 EmbeddingProvider, 

620 values_callable=lambda obj: [e.value for e in obj], 

621 ), 

622 nullable=False, 

623 ) 

624 embedding_dimension = Column(Integer, nullable=True) # Vector dimension 

625 

626 # Document metadata (for context) 

627 document_title = Column(Text, nullable=True) # Title of source document 

628 document_metadata = Column( 

629 JSON, nullable=True 

630 ) # Additional metadata from source 

631 

632 # Timestamps 

633 created_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

634 last_accessed = Column(UtcDateTime, nullable=True) 

635 

636 # Indexes for efficient queries. 

637 # NOTE: the old UniqueConstraint("chunk_hash", "collection_name", 

638 # name="uix_chunk_collection") was DROPPED (migration 0023). The RAG store 

639 # is now one-row-per-chunk keyed by the int PK (no cross-document chunk 

640 # sharing), so identical text in two documents legitimately produces two 

641 # rows; the unique constraint would raise IntegrityError on that (and on a 

642 # document with a repeated/blank chunk). chunk_hash stays a plain index for 

643 # query-time result de-duplication. 

644 __table_args__ = ( 

645 Index("idx_chunk_source", "source_type", "source_id"), 

646 Index("idx_chunk_collection", "collection_name", "created_at"), 

647 Index("idx_chunk_embedding", "embedding_id"), 

648 # AUTOINCREMENT so this id is NEVER reused after rows are deleted. 

649 # The id keys the vector store; a plain SQLite ROWID gets recycled, so 

650 # a recycled id could be re-adopted by an unrelated later chunk while a 

651 # stale vector for the old id still lingers in the index — making search 

652 # rehydrate the WRONG document's text. A monotonic id closes that. 

653 {"sqlite_autoincrement": True}, 

654 ) 

655 

656 def __repr__(self): 

657 return f"<DocumentChunk(collection='{self.collection_name}', source_type='{self.source_type}', index={self.chunk_index}, words={self.word_count})>" 

658 

659 

660class DownloadQueue(Base): 

661 """ 

662 Queue for pending document downloads. 

663 Renamed from LibraryDownloadQueue for consistency. 

664 """ 

665 

666 __tablename__ = "download_queue" 

667 

668 id = Column(Integer, primary_key=True, autoincrement=True) 

669 

670 # What to download 

671 resource_id = Column( 

672 Integer, 

673 ForeignKey("research_resources.id", ondelete="CASCADE"), 

674 nullable=False, 

675 unique=True, # One queue entry per resource 

676 ) 

677 research_id = Column(String(36), nullable=False, index=True) 

678 

679 # Target collection (defaults to Library collection) 

680 collection_id = Column( 

681 String(36), 

682 ForeignKey("collections.id", ondelete="SET NULL"), 

683 nullable=True, 

684 index=True, 

685 ) 

686 

687 # Queue management 

688 priority = Column(Integer, default=0) # Higher = more important 

689 status = Column( 

690 Enum( 

691 DocumentStatus, values_callable=lambda obj: [e.value for e in obj] 

692 ), 

693 nullable=False, 

694 default=DocumentStatus.PENDING, 

695 ) 

696 attempts = Column(Integer, default=0) 

697 max_attempts = Column(Integer, default=3) 

698 

699 # Error tracking 

700 last_error = Column(Text, nullable=True) 

701 last_attempt_at = Column(UtcDateTime, nullable=True) 

702 

703 # Timestamps 

704 queued_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

705 completed_at = Column(UtcDateTime, nullable=True) 

706 

707 # Relationships 

708 resource = relationship("ResearchResource", backref="download_queue") 

709 collection = relationship("Collection", backref="download_queue_items") 

710 

711 # Index for the common filter_by(research_id=..., status=...) query 

712 # pattern used by download_bulk, queue_all_undownloaded, and the 

713 # background scheduler. Mirrors Document.idx_research_documents. 

714 __table_args__ = ( 

715 Index("idx_download_queue_research_status", "research_id", "status"), 

716 ) 

717 

718 def __repr__(self): 

719 return f"<DownloadQueue(resource_id={self.resource_id}, status={self.status}, attempts={self.attempts})>" 

720 

721 

722class LibraryStatistics(Base): 

723 """ 

724 Aggregate statistics for the library. 

725 Updated periodically for dashboard display. 

726 """ 

727 

728 __tablename__ = "library_statistics" 

729 

730 id = Column(Integer, primary_key=True, autoincrement=True) 

731 

732 # Document counts 

733 total_documents = Column(Integer, default=0) 

734 total_pdfs = Column(Integer, default=0) 

735 total_html = Column(Integer, default=0) 

736 total_other = Column(Integer, default=0) 

737 

738 # Storage metrics 

739 total_size_bytes = Column(Integer, default=0) 

740 average_document_size = Column(Integer, default=0) 

741 

742 # Research metrics 

743 total_researches_with_downloads = Column(Integer, default=0) 

744 average_documents_per_research = Column(Integer, default=0) 

745 

746 # Download metrics 

747 total_download_attempts = Column(Integer, default=0) 

748 successful_downloads = Column(Integer, default=0) 

749 failed_downloads = Column(Integer, default=0) 

750 pending_downloads = Column(Integer, default=0) 

751 

752 # Academic sources breakdown 

753 arxiv_count = Column(Integer, default=0) 

754 pubmed_count = Column(Integer, default=0) 

755 doi_count = Column(Integer, default=0) 

756 other_count = Column(Integer, default=0) 

757 

758 # Timestamps 

759 calculated_at = Column(UtcDateTime, default=utcnow(), nullable=False) 

760 

761 def __repr__(self): 

762 return f"<LibraryStatistics(documents={self.total_documents}, size={self.total_size_bytes})>" 

763 

764 

765class RAGIndex(Base): 

766 """ 

767 Tracks FAISS indices for RAG collections. 

768 Each full embedding and chunking configuration has its own FAISS index. 

769 """ 

770 

771 __tablename__ = "rag_indices" 

772 

773 id: Mapped[int] = mapped_column( 

774 Integer, primary_key=True, autoincrement=True 

775 ) 

776 

777 # Collection and model identification 

778 collection_name: Mapped[str] = mapped_column( 

779 String(100), nullable=False, index=True 

780 ) # 'collection_<uuid>' 

781 embedding_model: Mapped[str] = mapped_column( 

782 String(100), nullable=False 

783 ) # e.g., 'all-MiniLM-L6-v2' 

784 embedding_model_type: Mapped[EmbeddingProvider] = mapped_column( 

785 Enum( 

786 EmbeddingProvider, 

787 values_callable=lambda obj: [e.value for e in obj], 

788 ), 

789 nullable=False, 

790 ) 

791 embedding_dimension: Mapped[int] = mapped_column( 

792 Integer, nullable=False 

793 ) # Vector dimension 

794 

795 # Index file location 

796 index_path: Mapped[str] = mapped_column( 

797 Text, nullable=False 

798 ) # Path to .faiss file 

799 index_hash: Mapped[str] = mapped_column( 

800 String(64), nullable=False, unique=True, index=True 

801 ) # SHA256 of collection+model for uniqueness 

802 

803 # Chunking parameters used 

804 chunk_size: Mapped[int] = mapped_column(Integer, nullable=False) 

805 chunk_overlap: Mapped[int] = mapped_column(Integer, nullable=False) 

806 

807 # Advanced embedding configuration options (Issue #1054) 

808 splitter_type: Mapped[str | None] = mapped_column( 

809 String(50), nullable=True 

810 ) # Splitter type: 'recursive', 'semantic', 'token', 'sentence' 

811 text_separators: Mapped[list[str] | None] = mapped_column( 

812 JSON, nullable=True 

813 ) # Text separators for chunking, e.g., ["\n\n", "\n", ". ", " ", ""] 

814 distance_metric: Mapped[str | None] = mapped_column( 

815 String(50), nullable=True 

816 ) # Distance metric: 'cosine', 'l2', 'dot_product' 

817 normalize_vectors: Mapped[bool | None] = mapped_column( 

818 Boolean, nullable=True 

819 ) # Whether to normalize embeddings with L2 

820 index_type: Mapped[str | None] = mapped_column( 

821 String(50), nullable=True 

822 ) # FAISS index type: 'flat', 'hnsw', 'ivf' 

823 

824 # Index statistics 

825 chunk_count: Mapped[int | None] = mapped_column( 

826 Integer, default=0 

827 ) # Number of chunks in this index 

828 total_documents: Mapped[int | None] = mapped_column( 

829 Integer, default=0 

830 ) # Number of source documents 

831 

832 # Status 

833 status: Mapped[RAGIndexStatus] = mapped_column( 

834 Enum( 

835 RAGIndexStatus, values_callable=lambda obj: [e.value for e in obj] 

836 ), 

837 nullable=False, 

838 default=RAGIndexStatus.ACTIVE, 

839 ) 

840 is_current: Mapped[bool | None] = mapped_column( 

841 Boolean, default=True 

842 ) # Whether this is the current index for this collection 

843 

844 # Timestamps 

845 created_at: Mapped[datetime] = mapped_column( 

846 UtcDateTime, default=utcnow(), nullable=False 

847 ) 

848 last_updated_at: Mapped[datetime] = mapped_column( 

849 UtcDateTime, default=utcnow(), onupdate=utcnow(), nullable=False 

850 ) 

851 last_used_at: Mapped[datetime | None] = mapped_column( 

852 UtcDateTime, nullable=True 

853 ) # Last time index was searched 

854 

855 __table_args__ = ( 

856 Index("idx_collection_current", "collection_name", "is_current"), 

857 ) 

858 

859 def __repr__(self): 

860 return f"<RAGIndex(collection='{self.collection_name}', model='{self.embedding_model}', chunks={self.chunk_count})>" 

861 

862 

863class RagDocumentStatus(Base): 

864 """ 

865 Tracks which documents have been indexed for RAG. 

866 Row existence = document is indexed. No row = not indexed. 

867 Simple and avoids ORM caching issues.