Coverage for src/local_deep_research/web_search_engines/engines/search_engine_paperless.py: 92%
316 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""
2Paperless-ngx search engine implementation for Local Deep Research.
4This module provides a proper search engine implementation that connects to a Paperless-ngx
5instance, allowing LDR to search and retrieve documents from your personal
6document management system.
7"""
9import re
10from typing import Any, Dict, List, Optional
11import requests
12from urllib.parse import urljoin
14from langchain_core.language_models import BaseLLM
15from ...security.secure_logging import logger
17from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
18from ...security import redact_url_for_log, safe_get
19from ...utilities.llm_utils import invoke_llm_sync
22class PaperlessSearchEngine(BaseSearchEngine):
23 """Paperless-ngx search engine implementation with full LDR integration."""
25 is_local = True
26 is_lexical = True
27 needs_llm_relevance_filter = True
28 # Egress (ADR-0007): a local document store — sensitive, contained. The
29 # url_setting fail-up reclassifies exposure to EXPOSING when api_url is a
30 # public host (quadrant 4: usable only by itself, contained inference).
31 egress_sensitivity = Sensitivity.SENSITIVE
32 egress_exposure = Exposure.CONTAINED
33 # secrets to redact from error messages (see BaseSearchEngine._scrub_error)
34 _secret_attrs = ("api_token",)
35 # url_setting feeds the PDP's fail-up URL override: if the configured
36 # api_url resolves to a PUBLIC host, the engine is reclassified public
37 # so PRIVATE_ONLY denies it at selection time (queries would leave the
38 # box). A local api_url keeps the static is_local classification —
39 # the override only ever tightens, never relaxes.
40 url_setting = "search.engine.web.paperless.default_params.api_url"
42 # Class constants for magic numbers
43 MAX_SNIPPET_LENGTH = 3000 # Reasonable limit to avoid context window issues
44 SNIPPET_CONTEXT_BEFORE = 500 # Characters before matched term in snippet
45 SNIPPET_CONTEXT_AFTER = 2500 # Characters after matched term in snippet
47 def __init__(
48 self,
49 api_url: str | None = None,
50 api_key: str | None = None,
51 api_token: str
52 | None = None, # Support both for backwards compatibility
53 max_results: int = 10,
54 timeout: int = 30,
55 verify_ssl: bool = True,
56 include_content: bool = True,
57 llm: Optional[BaseLLM] = None,
58 settings_snapshot: Optional[Dict[str, Any]] = None,
59 **kwargs,
60 ):
61 """
62 Initialize the Paperless-ngx search engine.
64 Args:
65 api_url: Base URL of Paperless-ngx instance (e.g., "http://localhost:8000")
66 If not provided, will look for PAPERLESS_API_URL env var
67 api_key: API token for authentication (preferred parameter name)
68 api_token: API token for authentication (backwards compatibility)
69 If not provided, will look for PAPERLESS_API_TOKEN env var
70 max_results: Maximum number of search results
71 timeout: Request timeout in seconds
72 verify_ssl: Whether to verify SSL certificates
73 include_content: Whether to include document content in results
74 llm: Language model for relevance filtering (optional)
75 settings_snapshot: Settings snapshot for thread context
76 **kwargs: Additional parameters passed to parent
77 """
78 super().__init__(
79 max_results=max_results,
80 llm=llm,
81 settings_snapshot=settings_snapshot,
82 **kwargs,
83 )
85 # Use provided configuration or get from settings
86 self.api_url = api_url
87 # Support both api_key and api_token for compatibility
88 self.api_token = api_key or api_token
90 # If no API URL provided, try to get from settings_snapshot
91 if not self.api_url and settings_snapshot:
92 self.api_url = settings_snapshot.get(
93 "search.engine.web.paperless.default_params.api_url",
94 "http://localhost:8000",
95 )
97 # If no API token provided, try to get from settings_snapshot
98 if not self.api_token and settings_snapshot:
99 self.api_token = settings_snapshot.get(
100 "search.engine.web.paperless.api_key", ""
101 )
103 # Fix AttributeError: Check if api_url is None before calling rstrip
104 if self.api_url:
105 # Remove trailing slash from API URL
106 self.api_url = self.api_url.rstrip("/")
107 else:
108 # Default to localhost if nothing provided
109 self.api_url = "http://localhost:8000"
110 logger.warning(
111 "No Paperless API URL provided, using default: http://localhost:8000"
112 )
114 self.timeout = timeout
115 self.verify_ssl = verify_ssl
116 self.include_content = include_content
118 # Set up headers for authentication
119 self.headers = {}
120 if self.api_token:
121 self.headers["Authorization"] = f"Token {self.api_token}"
123 logger.info(
124 f"Initialized Paperless-ngx search engine for {redact_url_for_log(self.api_url)}"
125 )
127 def _make_request(
128 self, endpoint: str, params: Optional[Dict] = None
129 ) -> Dict[str, Any]:
130 """
131 Make a request to the Paperless-ngx API.
133 Args:
134 endpoint: API endpoint path
135 params: Query parameters
137 Returns:
138 JSON response from the API
139 """
140 url = urljoin(self.api_url or "", endpoint)
142 logger.debug(f"Making request to: {redact_url_for_log(url)}")
143 logger.debug(
144 f"Request params: {sorted(params.keys()) if params else None}"
145 )
146 logger.debug(
147 f"Headers: {self.headers.keys() if self.headers else 'None'}"
148 )
150 try:
151 # Paperless is typically a local/private network service
152 response = safe_get(
153 url,
154 params=params,
155 headers=self.headers,
156 timeout=self.timeout,
157 verify=self.verify_ssl,
158 allow_private_ips=True,
159 allow_localhost=True,
160 )
161 response.raise_for_status()
162 result = response.json()
164 # Log response details
165 if isinstance(result, dict): 165 ↛ 183line 165 didn't jump to line 183 because the condition on line 165 was always true
166 if "results" in result:
167 logger.info(
168 f"API returned {len(result.get('results', []))} results, total count: {result.get('count', 'unknown')}"
169 )
170 # Log first result details if available
171 if result.get("results"):
172 first = result["results"][0]
173 logger.debug(
174 f"First result: id={first.get('id')}, title='{first.get('title', 'No title')[:50]}...'"
175 )
176 if "__search_hit__" in first:
177 logger.debug(
178 f"Has search hit data with score={first['__search_hit__'].get('score')}"
179 )
180 else:
181 logger.debug(f"API response keys: {result.keys()}")
183 return result # type: ignore[no-any-return]
184 except requests.exceptions.RequestException as e:
185 safe_msg = self._scrub_error(e)
186 logger.warning(f"Error making request to Paperless-ngx: {safe_msg}")
187 logger.debug(
188 f"Failed URL: {redact_url_for_log(url)}, "
189 f"params: {sorted(params.keys()) if params else None}"
190 )
191 return {}
193 def _expand_query_with_llm(self, query: str) -> str:
194 """
195 Use LLM to expand query with relevant keywords and synonyms.
197 Args:
198 query: Original search query
200 Returns:
201 Expanded query with keywords
202 """
203 if not self.llm:
204 logger.info("No LLM available for query expansion, using original")
205 return query
207 try:
208 prompt = f"""Paperless-ngx uses TF-IDF keyword search, not semantic search.
209Convert this query into keywords that would appear in documents.
211Query: "{query}"
213Output format: keyword1 OR keyword2 OR "multi word phrase" OR keyword3
214Include synonyms, plural forms, and technical terms.
216IMPORTANT: Output ONLY the search query. No explanations, no additional text."""
218 logger.debug(
219 f"Sending query expansion prompt to LLM ({len(query)} chars)"
220 )
221 response = invoke_llm_sync(self.llm, prompt)
222 expanded = (
223 str(response.content)
224 if hasattr(response, "content")
225 else str(response)
226 ).strip()
228 logger.debug(f"Raw LLM response received ({len(expanded)} chars)")
230 # Clean up the response - remove any explanatory text
231 if "\n" in expanded: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true
232 expanded = expanded.split("\n")[0]
233 logger.debug("Took first line of LLM response")
235 # Always trust the LLM's expansion - it knows better than hard-coded rules
236 logger.info(
237 f"LLM expanded query from {len(query)} chars to {len(expanded)} "
238 f"chars with {expanded.count('OR')} ORs"
239 )
240 logger.debug(f"Expanded query is {len(expanded)} chars")
241 return expanded
243 except Exception as e:
244 safe_msg = self._scrub_error(e)
245 logger.warning(f"Failed to expand query with LLM: {safe_msg}")
246 return query
248 def _multi_pass_search(self, query: str) -> List[Dict[str, Any]]:
249 """
250 Perform multiple search passes with different strategies.
252 Args:
253 query: Original search query
255 Returns:
256 Combined and deduplicated results
257 """
258 logger.info("Starting multi-pass search")
259 all_results = {} # Use dict to deduplicate by doc_id
261 # Pass 1: Original query
262 params = {
263 "query": query,
264 "page_size": self.max_results,
265 "ordering": "-score",
266 }
268 logger.info(
269 f"Pass 1 - Original query ({len(query)} chars, "
270 f"max_results={self.max_results})"
271 )
272 response = self._make_request("/api/documents/", params=params)
274 if response and "results" in response:
275 pass1_count = len(response["results"])
276 logger.info(f"Pass 1 returned {pass1_count} documents")
277 for doc in response["results"]:
278 doc_id = doc.get("id")
279 if doc_id and doc_id not in all_results: 279 ↛ 277line 279 didn't jump to line 277 because the condition on line 279 was always true
280 all_results[doc_id] = doc
281 logger.debug(
282 f"Added doc {doc_id}: {doc.get('title', 'No title')}"
283 )
284 else:
285 logger.warning(
286 f"Pass 1 returned no results or invalid response: {response}"
287 )
289 # Pass 2: LLM-expanded keywords (if LLM available)
290 if self.llm:
291 expanded_query = self._expand_query_with_llm(query)
292 if expanded_query != query: 292 ↛ 320line 292 didn't jump to line 320 because the condition on line 292 was always true
293 params["query"] = expanded_query
294 params["page_size"] = self.max_results * 2 # Get more results
296 logger.info(
297 f"Pass 2 - Using expanded query with {expanded_query.count('OR')} ORs"
298 )
299 logger.info(
300 f"Pass 2 - Max results set to: {params['page_size']}"
301 )
302 response = self._make_request("/api/documents/", params=params)
304 if response and "results" in response: 304 ↛ 318line 304 didn't jump to line 318 because the condition on line 304 was always true
305 pass2_new = 0
306 for doc in response["results"]:
307 doc_id = doc.get("id")
308 if doc_id and doc_id not in all_results:
309 all_results[doc_id] = doc
310 pass2_new += 1
311 logger.debug(
312 f"Pass 2 added new doc {doc_id}: {doc.get('title', 'No title')}"
313 )
314 logger.info(
315 f"Pass 2 found {len(response['results'])} docs, added {pass2_new} new"
316 )
317 else:
318 logger.warning("Pass 2 returned no results")
319 else:
320 logger.info("Pass 2 skipped - expanded query same as original")
321 else:
322 logger.info("Pass 2 skipped - no LLM available")
324 # Sort by relevance score if available
325 logger.info(f"Total unique documents collected: {len(all_results)}")
326 sorted_results = sorted(
327 all_results.values(),
328 key=lambda x: x.get("__search_hit__", {}).get("score", 0),
329 reverse=True,
330 )
332 final_results = sorted_results[: self.max_results]
333 logger.info(
334 f"Returning top {len(final_results)} documents after sorting by score"
335 )
337 # Log titles and scores of final results
338 for i, doc in enumerate(final_results[:5], 1): # Log first 5
339 score = doc.get("__search_hit__", {}).get("score", 0)
340 logger.debug(
341 f"Result {i}: '{doc.get('title', 'No title')}' (score={score})"
342 )
344 return final_results
346 def _get_previews(self, query: str) -> List[Dict[str, Any]]:
347 """
348 Get preview results from Paperless-ngx using multi-pass strategy.
350 Args:
351 query: Search query
353 Returns:
354 List of preview dictionaries
355 """
356 try:
357 # Use multi-pass search strategy
358 results = self._multi_pass_search(query)
360 if not results:
361 return []
363 # Convert documents to preview format
364 # Note: Each document may return multiple previews (one per highlight)
365 previews = []
366 for doc_data in results:
367 doc_previews = self._convert_document_to_preview(
368 doc_data, query
369 )
370 # Handle both single preview and list of previews
371 if isinstance(doc_previews, list): 371 ↛ 372line 371 didn't jump to line 372 because the condition on line 371 was never true
372 previews.extend(doc_previews)
373 else:
374 previews.append(doc_previews)
376 logger.info(f"Found {len(previews)} documents in Paperless-ngx")
377 return previews
379 except Exception as e:
380 safe_msg = self._scrub_error(e)
381 logger.warning(
382 f"Error getting previews from Paperless-ngx: {safe_msg}"
383 )
384 return []
386 def _convert_document_to_preview(
387 self, doc_data: Dict[str, Any], query: str = ""
388 ) -> Dict[str, Any] | List[Dict[str, Any]]:
389 """
390 Convert a Paperless-ngx document to LDR preview format.
392 Args:
393 doc_data: Document data from the API
394 query: Original search query (for context)
396 Returns:
397 Preview dictionary in LDR format
398 """
399 # Extract title
400 title = doc_data.get("title", f"Document {doc_data.get('id')}")
401 doc_id = doc_data.get("id")
403 logger.info(
404 f"Converting document {doc_id}: '{title}' to preview format"
405 )
407 # Build URL - use the web interface URL for user access
408 url = f"{self.api_url}/documents/{doc_id}/details"
409 logger.debug(
410 f"Generated URL for doc {doc_id}: {redact_url_for_log(url)}"
411 )
413 # Extract snippet - prefer highlighted content from search
414 snippet = ""
415 search_score = 0.0
416 search_rank = None
417 all_highlights = [] # Initialize empty highlights list
419 if "__search_hit__" in doc_data:
420 search_hit = doc_data["__search_hit__"]
421 logger.debug(
422 f"Found __search_hit__ data for doc {doc_id}: score={search_hit.get('score')}, rank={search_hit.get('rank')}"
423 )
425 # Get highlights - this is the search snippet with matched terms
426 if search_hit.get("highlights"): 426 ↛ 479line 426 didn't jump to line 479 because the condition on line 426 was always true
427 # Highlights can be a string or list
428 highlights = search_hit.get("highlights")
429 logger.info(
430 f"Found highlights for doc {doc_id}: type={type(highlights).__name__}, length={len(str(highlights))}"
431 )
433 if isinstance(highlights, list):
434 logger.debug(
435 f"Highlights is list with {len(highlights)} items"
436 )
437 # IMPORTANT: Store highlights list for processing later
438 # Each highlight will become a separate search result for proper citation
439 all_highlights = highlights
440 # Use first highlight for the default snippet
441 snippet = highlights[0] if highlights else ""
442 logger.info(
443 f"Will create {len(highlights)} separate results from highlights"
444 )
445 else:
446 all_highlights = [
447 str(highlights)
448 ] # Single highlight as list
449 snippet = str(highlights)
451 logger.debug(
452 f"Raw snippet before cleaning (first 200 chars): {snippet[:200]}"
453 )
455 # Clean HTML tags but preserve the matched text
456 snippet = re.sub(r"<span[^>]*>", "**", snippet)
457 snippet = re.sub(r"</span>", "**", snippet)
458 snippet = re.sub(r"<[^>]+>", "", snippet)
460 logger.debug(
461 f"Cleaned snippet (first 200 chars): {snippet[:200]}"
462 )
464 # Limit snippet length to avoid context window issues
465 if ( 465 ↛ 470line 465 didn't jump to line 470 because the condition on line 465 was never true
466 self.MAX_SNIPPET_LENGTH
467 and len(snippet) > self.MAX_SNIPPET_LENGTH
468 ):
469 # Cut at word boundary to avoid mid-word truncation
470 snippet = (
471 snippet[: self.MAX_SNIPPET_LENGTH].rsplit(" ", 1)[0]
472 + "..."
473 )
474 logger.debug(
475 f"Truncated snippet to {self.MAX_SNIPPET_LENGTH} chars"
476 )
478 # Get search relevance metadata
479 search_score = search_hit.get("score", 0.0)
480 search_rank = search_hit.get("rank")
481 logger.info(
482 f"Search metadata for doc {doc_id}: score={search_score}, rank={search_rank}"
483 )
484 else:
485 logger.warning(
486 f"No __search_hit__ data for doc {doc_id}, will use content fallback"
487 )
489 if not snippet:
490 logger.info(
491 f"No snippet from highlights for doc {doc_id}, using content fallback"
492 )
493 # Fallback to content preview if no highlights available
494 content = doc_data.get("content", "")
495 if content: 495 ↛ 538line 495 didn't jump to line 538 because the condition on line 495 was always true
496 logger.debug(f"Document has content of length {len(content)}")
497 # Try to find context around query terms if possible
498 if query: 498 ↛ 530line 498 didn't jump to line 530 because the condition on line 498 was always true
499 query_terms = query.lower().split()
500 content_lower = content.lower()
501 logger.debug(
502 f"Searching content for {len(query_terms)} query terms"
503 )
505 # Find first occurrence of any query term
506 best_pos = -1
507 for term in query_terms:
508 pos = content_lower.find(term)
509 if pos != -1 and (best_pos == -1 or pos < best_pos): 509 ↛ 507line 509 didn't jump to line 507 because the condition on line 509 was always true
510 best_pos = pos
511 logger.debug(
512 f"Found a query term at position {pos}"
513 )
515 if best_pos != -1: 515 ↛ 525line 515 didn't jump to line 525 because the condition on line 515 was always true
516 # Extract context around the found term - much larger context for research
517 start = max(0, best_pos - 2000)
518 end = min(len(content), best_pos + 8000)
519 snippet = "..." + content[start:end] + "..."
520 logger.info(
521 f"Extracted snippet around query term at position {best_pos}"
522 )
523 else:
524 # Just take the beginning - use 10000 chars for research
525 snippet = content[:10000]
526 logger.info(
527 "No query terms found, using first 10000 chars of content"
528 )
529 else:
530 snippet = content[:10000]
531 logger.info(
532 "No query provided, using first 10000 chars of content"
533 )
535 if len(content) > 10000: 535 ↛ 536line 535 didn't jump to line 536 because the condition on line 535 was never true
536 snippet += "..."
537 else:
538 logger.warning(f"No content available for doc {doc_id}")
540 logger.info(f"Final snippet for doc {doc_id} has length {len(snippet)}")
542 # Build metadata
543 metadata = {
544 "doc_id": str(doc_id),
545 "correspondent": doc_data.get("correspondent_name", ""),
546 "document_type": doc_data.get("document_type_name", ""),
547 "created": doc_data.get("created", ""),
548 "modified": doc_data.get("modified", ""),
549 "archive_serial_number": doc_data.get("archive_serial_number"),
550 "search_score": search_score,
551 "search_rank": search_rank,
552 }
554 # Add tags if present
555 tags = doc_data.get("tags_list", [])
556 if isinstance(tags, list) and tags:
557 metadata["tags"] = ", ".join(str(tag) for tag in tags)
559 # Build enhanced title with available metadata for better citations
560 title_parts = []
562 # Add correspondent/author if available
563 correspondent = doc_data.get("correspondent_name", "")
564 if correspondent:
565 title_parts.append(f"{correspondent}.")
566 logger.debug(f"Added correspondent to title: {correspondent}")
568 # Add the document title
569 title_parts.append(title)
571 # Add document type if it's meaningful (not just generic types)
572 doc_type = doc_data.get("document_type_name", "")
573 if doc_type and doc_type not in ["Letter", "Other", "Document", ""]:
574 title_parts.append(f"({doc_type})")
575 logger.debug(f"Added document type to title: {doc_type}")
577 # Add year from created date if available
578 created_date = doc_data.get("created", "")
579 if created_date and len(created_date) >= 4:
580 year = created_date[:4]
581 title_parts.append(year)
582 logger.debug(f"Added year to title: {year}")
584 # Format the enhanced title for display in sources list
585 if title_parts: 585 ↛ 588line 585 didn't jump to line 588 because the condition on line 585 was always true
586 enhanced_title = " ".join(title_parts)
587 else:
588 enhanced_title = title
590 logger.info(f"Enhanced title for doc {doc_id}: '{enhanced_title}'")
592 # Build the preview
593 preview = {
594 "title": enhanced_title, # Use enhanced title with bibliographic info
595 "url": url,
596 "link": url, # Add 'link' key for compatibility with search utilities
597 "snippet": snippet,
598 "author": doc_data.get("correspondent_name", ""),
599 "date": doc_data.get("created", ""),
600 "source": "Paperless", # Keep source as the system name like other engines
601 "metadata": metadata,
602 "_raw_data": doc_data, # Store raw data for full content retrieval
603 }
605 logger.info(
606 f"Built preview for doc {doc_id}: URL={url}, snippet_len={len(snippet)}, has_author={bool(preview['author'])}, has_date={bool(preview['date'])}"
607 )
609 # Check if we have multiple highlights to return as separate results
610 if len(all_highlights) > 1:
611 # Create multiple previews, one for each highlight
612 previews = []
613 for i, highlight in enumerate(all_highlights):
614 # Clean each highlight
615 clean_snippet = re.sub(r"<span[^>]*>", "**", str(highlight))
616 clean_snippet = re.sub(r"</span>", "**", clean_snippet)
617 clean_snippet = re.sub(r"<[^>]+>", "", clean_snippet)
619 # Create a preview for this highlight
620 highlight_preview = {
621 "title": f"{enhanced_title} (excerpt {i + 1})", # Differentiate each excerpt
622 "url": url,
623 "link": url,
624 "snippet": clean_snippet,
625 "author": doc_data.get("correspondent_name", ""),
626 "date": doc_data.get("created", ""),
627 "source": "Paperless",
628 "metadata": {
629 **metadata,
630 "excerpt_number": i + 1,
631 "total_excerpts": len(all_highlights),
632 },
633 "_raw_data": doc_data,
634 }
635 previews.append(highlight_preview)
637 logger.info(
638 f"Created {len(previews)} separate previews from highlights for doc {doc_id}"
639 )
640 return previews
641 # Single preview (original behavior)
642 return preview
644 def _get_full_content(
645 self, relevant_items: List[Dict[str, Any]]
646 ) -> List[Dict[str, Any]]:
647 """
648 Get full content for relevant documents.
650 Args:
651 relevant_items: List of relevant preview dictionaries
653 Returns:
654 List of dictionaries with full content
655 """
656 if not self.include_content:
657 # If content inclusion is disabled, just return previews
658 return relevant_items
660 logger.info(f"Getting full content for {len(relevant_items)} documents")
661 results = []
662 for idx, item in enumerate(relevant_items):
663 try:
664 logger.info(
665 f"Processing document {idx + 1}: title='{item.get('title', 'No title')[:50]}...', url={item.get('url', 'No URL')}"
666 )
667 logger.debug(f"Document {idx + 1} keys: {item.keys()}")
668 logger.debug(
669 f"Document {idx + 1} has snippet of length: {len(item.get('snippet', ''))}"
670 )
672 # Get the full document content if we have the raw data
673 if "_raw_data" in item:
674 doc_data = item["_raw_data"]
675 full_content = doc_data.get("content", "")
677 if not full_content:
678 # Try to fetch the document details
679 doc_id = item["metadata"].get("doc_id")
680 if doc_id: 680 ↛ 689line 680 didn't jump to line 689 because the condition on line 680 was always true
681 detail_response = self._make_request(
682 f"/api/documents/{doc_id}/"
683 )
684 if detail_response:
685 full_content = detail_response.get(
686 "content", ""
687 )
689 item["full_content"] = full_content or item["snippet"]
690 item["full_content_is_complete"] = bool(full_content)
691 logger.info(
692 f"Document {idx + 1} full content length: {len(item['full_content'])}"
693 )
694 else:
695 # Fallback to snippet if no raw data
696 item["full_content"] = item["snippet"]
697 item["full_content_is_complete"] = False
698 logger.info(
699 f"Document {idx + 1} using snippet as full content (no raw data)"
700 )
702 # Log the final document structure for debugging citation issues
703 logger.info(
704 f"Document {idx + 1} final structure: title='{item.get('title', '')[:50]}...', has_link={bool(item.get('link'))}, has_url={bool(item.get('url'))}, source='{item.get('source', 'Unknown')}'"
705 )
707 # Remove the raw data from the result
708 item.pop("_raw_data", None)
709 results.append(item)
711 except Exception as e:
712 safe_msg = self._scrub_error(e)
713 logger.warning(
714 f"Error getting full content for document: {safe_msg}"
715 )
716 item["full_content"] = item["snippet"]
717 item["full_content_is_complete"] = False
718 item.pop("_raw_data", None)
719 results.append(item)
721 return results
723 def run(
724 self, query: str, research_context: Dict[str, Any] | None = None
725 ) -> List[Dict[str, Any]]:
726 """
727 Execute search on Paperless-ngx.
729 Args:
730 query: Search query
731 research_context: Context from previous research
733 Returns:
734 List of search results in LDR format
735 """
736 # Keep policy denials outside the search-error handler.
737 self._verify_egress_scope()
738 try:
739 # Get previews
740 previews = self._get_previews(query)
742 if not previews:
743 return []
745 # Apply LLM relevance filtering if enabled by the factory
746 enable_llm_filter = getattr(
747 self, "enable_llm_relevance_filter", False
748 )
749 if enable_llm_filter and self.llm: 749 ↛ 750line 749 didn't jump to line 750 because the condition on line 749 was never true
750 filtered_previews = self._filter_for_relevance(previews, query)
751 if not filtered_previews:
752 logger.info(
753 f"LLM relevance filter returned no results "
754 f"from {len(previews)} previews"
755 )
756 else:
757 filtered_previews = previews
759 # Get full content for relevant items (cache-aware so repeat
760 # URLs across quick phase / subsections / subagents share).
761 results = self._get_full_content_cached(filtered_previews)
763 logger.info(
764 f"Search completed successfully, returning {len(results)} results"
765 )
766 # Enhanced logging to track document structure for citation debugging
767 for i, r in enumerate(results[:3], 1):
768 logger.info(
769 f"Result {i}: title='{r.get('title', '')[:50]}...', "
770 f"has_full_content={bool(r.get('full_content'))}, "
771 f"full_content_len={len(r.get('full_content', ''))}, "
772 f"snippet_len={len(r.get('snippet', ''))}, "
773 f"url={r.get('url', '')[:50]}"
774 )
776 return results
778 except Exception as e:
779 safe_msg = self._scrub_error(e)
780 logger.warning(f"Error in Paperless-ngx search: {safe_msg}")
781 return []
783 async def arun(self, query: str) -> List[Dict[str, Any]]:
784 """
785 Async version of search.
787 Currently falls back to sync version.
788 """
789 return self.run(query)
791 def test_connection(self) -> bool:
792 """
793 Test the connection to Paperless-ngx.
795 Returns:
796 True if connection successful, False otherwise
797 """
798 try:
799 response = self._make_request("/api/")
800 return bool(response)
801 except Exception as e:
802 safe_msg = self._scrub_error(e)
803 logger.warning(f"Failed to connect to Paperless-ngx: {safe_msg}")
804 return False
806 def get_document_count(self) -> int:
807 """
808 Get the total number of documents in Paperless-ngx.
810 Returns:
811 Number of documents, or -1 if error
812 """
813 try:
814 response = self._make_request(
815 "/api/documents/", params={"page_size": 1}
816 )
817 return int(response.get("count", -1))
818 except Exception:
819 logger.debug("Failed to fetch document count", exc_info=True)
820 return -1