Coverage for src/local_deep_research/web_search_engines/engines/search_engine_paperless.py: 92%
312 statements
« prev ^ index » next coverage.py v7.15.1, created at 2026-07-20 01:24 +0000
1"""
2Paperless-ngx search engine implementation for Local Deep Research.
4This module provides a proper search engine implementation that connects to a Paperless-ngx
5instance, allowing LDR to search and retrieve documents from your personal
6document management system.
7"""
9import re
10from typing import Any, Dict, List, Optional
11import requests
12from urllib.parse import urljoin
14from langchain_core.language_models import BaseLLM
15from ...security.secure_logging import logger
17from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
18from ...security import redact_url_for_log, safe_get
21class PaperlessSearchEngine(BaseSearchEngine):
22 """Paperless-ngx search engine implementation with full LDR integration."""
24 is_local = True
25 is_lexical = True
26 needs_llm_relevance_filter = True
27 # Egress (ADR-0007): a local document store — sensitive, contained. The
28 # url_setting fail-up reclassifies exposure to EXPOSING when api_url is a
29 # public host (quadrant 4: usable only by itself, contained inference).
30 egress_sensitivity = Sensitivity.SENSITIVE
31 egress_exposure = Exposure.CONTAINED
32 # secrets to redact from error messages (see BaseSearchEngine._scrub_error)
33 _secret_attrs = ("api_token",)
34 # url_setting feeds the PDP's fail-up URL override: if the configured
35 # api_url resolves to a PUBLIC host, the engine is reclassified public
36 # so PRIVATE_ONLY denies it at selection time (queries would leave the
37 # box). A local api_url keeps the static is_local classification —
38 # the override only ever tightens, never relaxes.
39 url_setting = "search.engine.web.paperless.default_params.api_url"
41 # Class constants for magic numbers
42 MAX_SNIPPET_LENGTH = 3000 # Reasonable limit to avoid context window issues
43 SNIPPET_CONTEXT_BEFORE = 500 # Characters before matched term in snippet
44 SNIPPET_CONTEXT_AFTER = 2500 # Characters after matched term in snippet
46 def __init__(
47 self,
48 api_url: str | None = None,
49 api_key: str | None = None,
50 api_token: str
51 | None = None, # Support both for backwards compatibility
52 max_results: int = 10,
53 timeout: int = 30,
54 verify_ssl: bool = True,
55 include_content: bool = True,
56 llm: Optional[BaseLLM] = None,
57 settings_snapshot: Optional[Dict[str, Any]] = None,
58 **kwargs,
59 ):
60 """
61 Initialize the Paperless-ngx search engine.
63 Args:
64 api_url: Base URL of Paperless-ngx instance (e.g., "http://localhost:8000")
65 If not provided, will look for PAPERLESS_API_URL env var
66 api_key: API token for authentication (preferred parameter name)
67 api_token: API token for authentication (backwards compatibility)
68 If not provided, will look for PAPERLESS_API_TOKEN env var
69 max_results: Maximum number of search results
70 timeout: Request timeout in seconds
71 verify_ssl: Whether to verify SSL certificates
72 include_content: Whether to include document content in results
73 llm: Language model for relevance filtering (optional)
74 settings_snapshot: Settings snapshot for thread context
75 **kwargs: Additional parameters passed to parent
76 """
77 super().__init__(
78 max_results=max_results,
79 llm=llm,
80 settings_snapshot=settings_snapshot,
81 **kwargs,
82 )
84 # Use provided configuration or get from settings
85 self.api_url = api_url
86 # Support both api_key and api_token for compatibility
87 self.api_token = api_key or api_token
89 # If no API URL provided, try to get from settings_snapshot
90 if not self.api_url and settings_snapshot:
91 self.api_url = settings_snapshot.get(
92 "search.engine.web.paperless.default_params.api_url",
93 "http://localhost:8000",
94 )
96 # If no API token provided, try to get from settings_snapshot
97 if not self.api_token and settings_snapshot:
98 self.api_token = settings_snapshot.get(
99 "search.engine.web.paperless.api_key", ""
100 )
102 # Fix AttributeError: Check if api_url is None before calling rstrip
103 if self.api_url:
104 # Remove trailing slash from API URL
105 self.api_url = self.api_url.rstrip("/")
106 else:
107 # Default to localhost if nothing provided
108 self.api_url = "http://localhost:8000"
109 logger.warning(
110 "No Paperless API URL provided, using default: http://localhost:8000"
111 )
113 self.timeout = timeout
114 self.verify_ssl = verify_ssl
115 self.include_content = include_content
117 # Set up headers for authentication
118 self.headers = {}
119 if self.api_token:
120 self.headers["Authorization"] = f"Token {self.api_token}"
122 logger.info(
123 f"Initialized Paperless-ngx search engine for {redact_url_for_log(self.api_url)}"
124 )
126 def _make_request(
127 self, endpoint: str, params: Optional[Dict] = None
128 ) -> Dict[str, Any]:
129 """
130 Make a request to the Paperless-ngx API.
132 Args:
133 endpoint: API endpoint path
134 params: Query parameters
136 Returns:
137 JSON response from the API
138 """
139 url = urljoin(self.api_url or "", endpoint)
141 logger.debug(f"Making request to: {redact_url_for_log(url)}")
142 logger.debug(f"Request params: {params}")
143 logger.debug(
144 f"Headers: {self.headers.keys() if self.headers else 'None'}"
145 )
147 try:
148 # Paperless is typically a local/private network service
149 response = safe_get(
150 url,
151 params=params,
152 headers=self.headers,
153 timeout=self.timeout,
154 verify=self.verify_ssl,
155 allow_private_ips=True,
156 allow_localhost=True,
157 )
158 response.raise_for_status()
159 result = response.json()
161 # Log response details
162 if isinstance(result, dict): 162 ↛ 180line 162 didn't jump to line 180 because the condition on line 162 was always true
163 if "results" in result:
164 logger.info(
165 f"API returned {len(result.get('results', []))} results, total count: {result.get('count', 'unknown')}"
166 )
167 # Log first result details if available
168 if result.get("results"):
169 first = result["results"][0]
170 logger.debug(
171 f"First result: id={first.get('id')}, title='{first.get('title', 'No title')[:50]}...'"
172 )
173 if "__search_hit__" in first:
174 logger.debug(
175 f"Has search hit data with score={first['__search_hit__'].get('score')}"
176 )
177 else:
178 logger.debug(f"API response keys: {result.keys()}")
180 return result # type: ignore[no-any-return]
181 except requests.exceptions.RequestException as e:
182 safe_msg = self._scrub_error(e)
183 logger.warning(f"Error making request to Paperless-ngx: {safe_msg}")
184 logger.debug(
185 f"Failed URL: {redact_url_for_log(url)}, params: {params}"
186 )
187 return {}
189 def _expand_query_with_llm(self, query: str) -> str:
190 """
191 Use LLM to expand query with relevant keywords and synonyms.
193 Args:
194 query: Original search query
196 Returns:
197 Expanded query with keywords
198 """
199 if not self.llm:
200 logger.info(
201 f"No LLM available for query expansion, using original: '{query}'"
202 )
203 return query
205 try:
206 prompt = f"""Paperless-ngx uses TF-IDF keyword search, not semantic search.
207Convert this query into keywords that would appear in documents.
209Query: "{query}"
211Output format: keyword1 OR keyword2 OR "multi word phrase" OR keyword3
212Include synonyms, plural forms, and technical terms.
214IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
216 logger.debug(
217 f"Sending query expansion prompt to LLM for: '{query}'"
218 )
219 response = self.llm.invoke(prompt)
220 expanded = (
221 str(response.content)
222 if hasattr(response, "content")
223 else str(response)
224 ).strip()
226 logger.debug(
227 f"Raw LLM response (first 500 chars): {expanded[:500]}"
228 )
230 # Clean up the response - remove any explanatory text
231 if "\n" in expanded: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true
232 expanded = expanded.split("\n")[0]
233 logger.debug("Took first line of LLM response")
235 # Always trust the LLM's expansion - it knows better than hard-coded rules
236 logger.info(
237 f"LLM expanded query from '{query}' to {len(expanded)} chars with {expanded.count('OR')} ORs"
238 )
239 logger.debug(
240 f"Expanded query preview (first 200 chars): {expanded[:200]}..."
241 )
242 return expanded
244 except Exception as e:
245 safe_msg = self._scrub_error(e)
246 logger.warning(f"Failed to expand query with LLM: {safe_msg}")
247 return query
249 def _multi_pass_search(self, query: str) -> List[Dict[str, Any]]:
250 """
251 Perform multiple search passes with different strategies.
253 Args:
254 query: Original search query
256 Returns:
257 Combined and deduplicated results
258 """
259 logger.info(f"Starting multi-pass search for query: '{query}'")
260 all_results = {} # Use dict to deduplicate by doc_id
262 # Pass 1: Original query
263 params = {
264 "query": query,
265 "page_size": self.max_results,
266 "ordering": "-score",
267 }
269 logger.info(
270 f"Pass 1 - Original query: '{query}' (max_results={self.max_results})"
271 )
272 response = self._make_request("/api/documents/", params=params)
274 if response and "results" in response:
275 pass1_count = len(response["results"])
276 logger.info(f"Pass 1 returned {pass1_count} documents")
277 for doc in response["results"]:
278 doc_id = doc.get("id")
279 if doc_id and doc_id not in all_results: 279 ↛ 277line 279 didn't jump to line 277 because the condition on line 279 was always true
280 all_results[doc_id] = doc
281 logger.debug(
282 f"Added doc {doc_id}: {doc.get('title', 'No title')}"
283 )
284 else:
285 logger.warning(
286 f"Pass 1 returned no results or invalid response: {response}"
287 )
289 # Pass 2: LLM-expanded keywords (if LLM available)
290 if self.llm:
291 expanded_query = self._expand_query_with_llm(query)
292 if expanded_query != query: 292 ↛ 323line 292 didn't jump to line 323 because the condition on line 292 was always true
293 params["query"] = expanded_query
294 params["page_size"] = self.max_results * 2 # Get more results
296 logger.info(
297 f"Pass 2 - Using expanded query with {expanded_query.count('OR')} ORs"
298 )
299 logger.debug(
300 f"Pass 2 - Full expanded query (first 500 chars): '{expanded_query[:500]}...'"
301 )
302 logger.info(
303 f"Pass 2 - Max results set to: {params['page_size']}"
304 )
305 response = self._make_request("/api/documents/", params=params)
307 if response and "results" in response: 307 ↛ 321line 307 didn't jump to line 321 because the condition on line 307 was always true
308 pass2_new = 0
309 for doc in response["results"]:
310 doc_id = doc.get("id")
311 if doc_id and doc_id not in all_results:
312 all_results[doc_id] = doc
313 pass2_new += 1
314 logger.debug(
315 f"Pass 2 added new doc {doc_id}: {doc.get('title', 'No title')}"
316 )
317 logger.info(
318 f"Pass 2 found {len(response['results'])} docs, added {pass2_new} new"
319 )
320 else:
321 logger.warning("Pass 2 returned no results")
322 else:
323 logger.info("Pass 2 skipped - expanded query same as original")
324 else:
325 logger.info("Pass 2 skipped - no LLM available")
327 # Sort by relevance score if available
328 logger.info(f"Total unique documents collected: {len(all_results)}")
329 sorted_results = sorted(
330 all_results.values(),
331 key=lambda x: x.get("__search_hit__", {}).get("score", 0),
332 reverse=True,
333 )
335 final_results = sorted_results[: self.max_results]
336 logger.info(
337 f"Returning top {len(final_results)} documents after sorting by score"
338 )
340 # Log titles and scores of final results
341 for i, doc in enumerate(final_results[:5], 1): # Log first 5
342 score = doc.get("__search_hit__", {}).get("score", 0)
343 logger.debug(
344 f"Result {i}: '{doc.get('title', 'No title')}' (score={score})"
345 )
347 return final_results
349 def _get_previews(self, query: str) -> List[Dict[str, Any]]:
350 """
351 Get preview results from Paperless-ngx using multi-pass strategy.
353 Args:
354 query: Search query
356 Returns:
357 List of preview dictionaries
358 """
359 try:
360 # Use multi-pass search strategy
361 results = self._multi_pass_search(query)
363 if not results:
364 return []
366 # Convert documents to preview format
367 # Note: Each document may return multiple previews (one per highlight)
368 previews = []
369 for doc_data in results:
370 doc_previews = self._convert_document_to_preview(
371 doc_data, query
372 )
373 # Handle both single preview and list of previews
374 if isinstance(doc_previews, list): 374 ↛ 375line 374 didn't jump to line 375 because the condition on line 374 was never true
375 previews.extend(doc_previews)
376 else:
377 previews.append(doc_previews)
379 logger.info(
380 f"Found {len(previews)} documents in Paperless-ngx for query: {query}"
381 )
382 return previews
384 except Exception as e:
385 safe_msg = self._scrub_error(e)
386 logger.warning(
387 f"Error getting previews from Paperless-ngx: {safe_msg}"
388 )
389 return []
391 def _convert_document_to_preview(
392 self, doc_data: Dict[str, Any], query: str = ""
393 ) -> Dict[str, Any] | List[Dict[str, Any]]:
394 """
395 Convert a Paperless-ngx document to LDR preview format.
397 Args:
398 doc_data: Document data from the API
399 query: Original search query (for context)
401 Returns:
402 Preview dictionary in LDR format
403 """
404 # Extract title
405 title = doc_data.get("title", f"Document {doc_data.get('id')}")
406 doc_id = doc_data.get("id")
408 logger.info(
409 f"Converting document {doc_id}: '{title}' to preview format"
410 )
412 # Build URL - use the web interface URL for user access
413 url = f"{self.api_url}/documents/{doc_id}/details"
414 logger.debug(
415 f"Generated URL for doc {doc_id}: {redact_url_for_log(url)}"
416 )
418 # Extract snippet - prefer highlighted content from search
419 snippet = ""
420 search_score = 0.0
421 search_rank = None
422 all_highlights = [] # Initialize empty highlights list
424 if "__search_hit__" in doc_data:
425 search_hit = doc_data["__search_hit__"]
426 logger.debug(
427 f"Found __search_hit__ data for doc {doc_id}: score={search_hit.get('score')}, rank={search_hit.get('rank')}"
428 )
430 # Get highlights - this is the search snippet with matched terms
431 if search_hit.get("highlights"): 431 ↛ 484line 431 didn't jump to line 484 because the condition on line 431 was always true
432 # Highlights can be a string or list
433 highlights = search_hit.get("highlights")
434 logger.info(
435 f"Found highlights for doc {doc_id}: type={type(highlights).__name__}, length={len(str(highlights))}"
436 )
438 if isinstance(highlights, list):
439 logger.debug(
440 f"Highlights is list with {len(highlights)} items"
441 )
442 # IMPORTANT: Store highlights list for processing later
443 # Each highlight will become a separate search result for proper citation
444 all_highlights = highlights
445 # Use first highlight for the default snippet
446 snippet = highlights[0] if highlights else ""
447 logger.info(
448 f"Will create {len(highlights)} separate results from highlights"
449 )
450 else:
451 all_highlights = [
452 str(highlights)
453 ] # Single highlight as list
454 snippet = str(highlights)
456 logger.debug(
457 f"Raw snippet before cleaning (first 200 chars): {snippet[:200]}"
458 )
460 # Clean HTML tags but preserve the matched text
461 snippet = re.sub(r"<span[^>]*>", "**", snippet)
462 snippet = re.sub(r"</span>", "**", snippet)
463 snippet = re.sub(r"<[^>]+>", "", snippet)
465 logger.debug(
466 f"Cleaned snippet (first 200 chars): {snippet[:200]}"
467 )
469 # Limit snippet length to avoid context window issues
470 if ( 470 ↛ 475line 470 didn't jump to line 475 because the condition on line 470 was never true
471 self.MAX_SNIPPET_LENGTH
472 and len(snippet) > self.MAX_SNIPPET_LENGTH
473 ):
474 # Cut at word boundary to avoid mid-word truncation
475 snippet = (
476 snippet[: self.MAX_SNIPPET_LENGTH].rsplit(" ", 1)[0]
477 + "..."
478 )
479 logger.debug(
480 f"Truncated snippet to {self.MAX_SNIPPET_LENGTH} chars"
481 )
483 # Get search relevance metadata
484 search_score = search_hit.get("score", 0.0)
485 search_rank = search_hit.get("rank")
486 logger.info(
487 f"Search metadata for doc {doc_id}: score={search_score}, rank={search_rank}"
488 )
489 else:
490 logger.warning(
491 f"No __search_hit__ data for doc {doc_id}, will use content fallback"
492 )
494 if not snippet:
495 logger.info(
496 f"No snippet from highlights for doc {doc_id}, using content fallback"
497 )
498 # Fallback to content preview if no highlights available
499 content = doc_data.get("content", "")
500 if content: 500 ↛ 543line 500 didn't jump to line 543 because the condition on line 500 was always true
501 logger.debug(f"Document has content of length {len(content)}")
502 # Try to find context around query terms if possible
503 if query: 503 ↛ 535line 503 didn't jump to line 535 because the condition on line 503 was always true
504 query_terms = query.lower().split()
505 content_lower = content.lower()
506 logger.debug(
507 f"Searching for query terms in content: {query_terms}"
508 )
510 # Find first occurrence of any query term
511 best_pos = -1
512 for term in query_terms:
513 pos = content_lower.find(term)
514 if pos != -1 and (best_pos == -1 or pos < best_pos): 514 ↛ 512line 514 didn't jump to line 512 because the condition on line 514 was always true
515 best_pos = pos
516 logger.debug(
517 f"Found term '{term}' at position {pos}"
518 )
520 if best_pos != -1: 520 ↛ 530line 520 didn't jump to line 530 because the condition on line 520 was always true
521 # Extract context around the found term - much larger context for research
522 start = max(0, best_pos - 2000)
523 end = min(len(content), best_pos + 8000)
524 snippet = "..." + content[start:end] + "..."
525 logger.info(
526 f"Extracted snippet around query term at position {best_pos}"
527 )
528 else:
529 # Just take the beginning - use 10000 chars for research
530 snippet = content[:10000]
531 logger.info(
532 "No query terms found, using first 10000 chars of content"
533 )
534 else:
535 snippet = content[:10000]
536 logger.info(
537 "No query provided, using first 10000 chars of content"
538 )
540 if len(content) > 10000: 540 ↛ 541line 540 didn't jump to line 541 because the condition on line 540 was never true
541 snippet += "..."
542 else:
543 logger.warning(f"No content available for doc {doc_id}")
545 logger.info(f"Final snippet for doc {doc_id} has length {len(snippet)}")
547 # Build metadata
548 metadata = {
549 "doc_id": str(doc_id),
550 "correspondent": doc_data.get("correspondent_name", ""),
551 "document_type": doc_data.get("document_type_name", ""),
552 "created": doc_data.get("created", ""),
553 "modified": doc_data.get("modified", ""),
554 "archive_serial_number": doc_data.get("archive_serial_number"),
555 "search_score": search_score,
556 "search_rank": search_rank,
557 }
559 # Add tags if present
560 tags = doc_data.get("tags_list", [])
561 if isinstance(tags, list) and tags:
562 metadata["tags"] = ", ".join(str(tag) for tag in tags)
564 # Build enhanced title with available metadata for better citations
565 title_parts = []
567 # Add correspondent/author if available
568 correspondent = doc_data.get("correspondent_name", "")
569 if correspondent:
570 title_parts.append(f"{correspondent}.")
571 logger.debug(f"Added correspondent to title: {correspondent}")
573 # Add the document title
574 title_parts.append(title)
576 # Add document type if it's meaningful (not just generic types)
577 doc_type = doc_data.get("document_type_name", "")
578 if doc_type and doc_type not in ["Letter", "Other", "Document", ""]:
579 title_parts.append(f"({doc_type})")
580 logger.debug(f"Added document type to title: {doc_type}")
582 # Add year from created date if available
583 created_date = doc_data.get("created", "")
584 if created_date and len(created_date) >= 4:
585 year = created_date[:4]
586 title_parts.append(year)
587 logger.debug(f"Added year to title: {year}")
589 # Format the enhanced title for display in sources list
590 if title_parts: 590 ↛ 593line 590 didn't jump to line 593 because the condition on line 590 was always true
591 enhanced_title = " ".join(title_parts)
592 else:
593 enhanced_title = title
595 logger.info(f"Enhanced title for doc {doc_id}: '{enhanced_title}'")
597 # Build the preview
598 preview = {
599 "title": enhanced_title, # Use enhanced title with bibliographic info
600 "url": url,
601 "link": url, # Add 'link' key for compatibility with search utilities
602 "snippet": snippet,
603 "author": doc_data.get("correspondent_name", ""),
604 "date": doc_data.get("created", ""),
605 "source": "Paperless", # Keep source as the system name like other engines
606 "metadata": metadata,
607 "_raw_data": doc_data, # Store raw data for full content retrieval
608 }
610 logger.info(
611 f"Built preview for doc {doc_id}: URL={url}, snippet_len={len(snippet)}, has_author={bool(preview['author'])}, has_date={bool(preview['date'])}"
612 )
614 # Check if we have multiple highlights to return as separate results
615 if len(all_highlights) > 1:
616 # Create multiple previews, one for each highlight
617 previews = []
618 for i, highlight in enumerate(all_highlights):
619 # Clean each highlight
620 clean_snippet = re.sub(r"<span[^>]*>", "**", str(highlight))
621 clean_snippet = re.sub(r"</span>", "**", clean_snippet)
622 clean_snippet = re.sub(r"<[^>]+>", "", clean_snippet)
624 # Create a preview for this highlight
625 highlight_preview = {
626 "title": f"{enhanced_title} (excerpt {i + 1})", # Differentiate each excerpt
627 "url": url,
628 "link": url,
629 "snippet": clean_snippet,
630 "author": doc_data.get("correspondent_name", ""),
631 "date": doc_data.get("created", ""),
632 "source": "Paperless",
633 "metadata": {
634 **metadata,
635 "excerpt_number": i + 1,
636 "total_excerpts": len(all_highlights),
637 },
638 "_raw_data": doc_data,
639 }
640 previews.append(highlight_preview)
642 logger.info(
643 f"Created {len(previews)} separate previews from highlights for doc {doc_id}"
644 )
645 return previews
646 # Single preview (original behavior)
647 return preview
649 def _get_full_content(
650 self, relevant_items: List[Dict[str, Any]]
651 ) -> List[Dict[str, Any]]:
652 """
653 Get full content for relevant documents.
655 Args:
656 relevant_items: List of relevant preview dictionaries
658 Returns:
659 List of dictionaries with full content
660 """
661 if not self.include_content:
662 # If content inclusion is disabled, just return previews
663 return relevant_items
665 logger.info(f"Getting full content for {len(relevant_items)} documents")
666 results = []
667 for idx, item in enumerate(relevant_items):
668 try:
669 logger.info(
670 f"Processing document {idx + 1}: title='{item.get('title', 'No title')[:50]}...', url={item.get('url', 'No URL')}"
671 )
672 logger.debug(f"Document {idx + 1} keys: {item.keys()}")
673 logger.debug(
674 f"Document {idx + 1} has snippet of length: {len(item.get('snippet', ''))}"
675 )
677 # Get the full document content if we have the raw data
678 if "_raw_data" in item:
679 doc_data = item["_raw_data"]
680 full_content = doc_data.get("content", "")
682 if not full_content:
683 # Try to fetch the document details
684 doc_id = item["metadata"].get("doc_id")
685 if doc_id: 685 ↛ 694line 685 didn't jump to line 694 because the condition on line 685 was always true
686 detail_response = self._make_request(
687 f"/api/documents/{doc_id}/"
688 )
689 if detail_response:
690 full_content = detail_response.get(
691 "content", ""
692 )
694 item["full_content"] = full_content or item["snippet"]
695 logger.info(
696 f"Document {idx + 1} full content length: {len(item['full_content'])}"
697 )
698 else:
699 # Fallback to snippet if no raw data
700 item["full_content"] = item["snippet"]
701 logger.info(
702 f"Document {idx + 1} using snippet as full content (no raw data)"
703 )
705 # Log the final document structure for debugging citation issues
706 logger.info(
707 f"Document {idx + 1} final structure: title='{item.get('title', '')[:50]}...', has_link={bool(item.get('link'))}, has_url={bool(item.get('url'))}, source='{item.get('source', 'Unknown')}'"
708 )
710 # Remove the raw data from the result
711 item.pop("_raw_data", None)
712 results.append(item)
714 except Exception as e:
715 safe_msg = self._scrub_error(e)
716 logger.warning(
717 f"Error getting full content for document: {safe_msg}"
718 )
719 item["full_content"] = item["snippet"]
720 item.pop("_raw_data", None)
721 results.append(item)
723 return results
725 def run(
726 self, query: str, research_context: Dict[str, Any] | None = None
727 ) -> List[Dict[str, Any]]:
728 """
729 Execute search on Paperless-ngx.
731 Args:
732 query: Search query
733 research_context: Context from previous research
735 Returns:
736 List of search results in LDR format
737 """
738 try:
739 # Get previews
740 previews = self._get_previews(query)
742 if not previews:
743 return []
745 # Apply LLM relevance filtering if enabled by the factory
746 enable_llm_filter = getattr(
747 self, "enable_llm_relevance_filter", False
748 )
749 if enable_llm_filter and self.llm: 749 ↛ 750line 749 didn't jump to line 750 because the condition on line 749 was never true
750 filtered_previews = self._filter_for_relevance(previews, query)
751 if not filtered_previews:
752 logger.info(
753 f"LLM relevance filter returned no results "
754 f"from {len(previews)} previews for query: {query}"
755 )
756 else:
757 filtered_previews = previews
759 # Get full content for relevant items
760 results = self._get_full_content(filtered_previews)
762 logger.info(
763 f"Search completed successfully, returning {len(results)} results"
764 )
765 # Enhanced logging to track document structure for citation debugging
766 for i, r in enumerate(results[:3], 1):
767 logger.info(
768 f"Result {i}: title='{r.get('title', '')[:50]}...', "
769 f"has_full_content={bool(r.get('full_content'))}, "
770 f"full_content_len={len(r.get('full_content', ''))}, "
771 f"snippet_len={len(r.get('snippet', ''))}, "
772 f"url={r.get('url', '')[:50]}"
773 )
775 return results
777 except Exception as e:
778 safe_msg = self._scrub_error(e)
779 logger.warning(f"Error in Paperless-ngx search: {safe_msg}")
780 return []
782 async def arun(self, query: str) -> List[Dict[str, Any]]:
783 """
784 Async version of search.
786 Currently falls back to sync version.
787 """
788 return self.run(query)
790 def test_connection(self) -> bool:
791 """
792 Test the connection to Paperless-ngx.
794 Returns:
795 True if connection successful, False otherwise
796 """
797 try:
798 response = self._make_request("/api/")
799 return bool(response)
800 except Exception as e:
801 safe_msg = self._scrub_error(e)
802 logger.warning(f"Failed to connect to Paperless-ngx: {safe_msg}")
803 return False
805 def get_document_count(self) -> int:
806 """
807 Get the total number of documents in Paperless-ngx.
809 Returns:
810 Number of documents, or -1 if error
811 """
812 try:
813 response = self._make_request(
814 "/api/documents/", params={"page_size": 1}
815 )
816 return int(response.get("count", -1))
817 except Exception:
818 logger.debug("Failed to fetch document count", exc_info=True)
819 return -1