Coverage for src/local_deep_research/llm/providers/implementations/llamacpp.py: 100%

1"""llama.cpp LLM provider for Local Deep Research. 

2 

3Talks to llama.cpp's OpenAI-compatible HTTP server (`llama-server`) instead 

4of loading models in-process via `llama-cpp-python`. Modeled after 

5`LMStudioProvider`. For setups that need API key auth or non-default URLs 

6beyond a single endpoint, use the `openai_endpoint` provider directly. 

7""" 

8 

9from ....config.constants import DEFAULT_LLAMACPP_URL 

10from ....utilities.url_utils import normalize_url 

11from ..base import Exposure 

12from ..openai_base import OpenAICompatibleProvider 

13 

14 

15class LlamaCppProvider(OpenAICompatibleProvider): 

16 """llama.cpp provider using its OpenAI-compatible HTTP endpoint. 

17 

18 Run `llama-server -m <model.gguf>` (port 8080 by default) and point 

19 `llm.llamacpp.url` at the server's `/v1` endpoint. 

20 """ 

21 

22 provider_name = "llama.cpp" 

23 # llama-server HAS an API key concept (for setups behind an auth proxy); 

24 # api_key_optional makes the base resolver fall back to a placeholder 

25 # when no key is set, instead of raising. 

26 api_key_setting = "llm.llamacpp.api_key" 

27 api_key_optional = True 

28 url_setting = "llm.llamacpp.url" # type: ignore[assignment] 

29 default_base_url = DEFAULT_LLAMACPP_URL 

30 default_model = "" # User must specify the model loaded by llama-server 

31 

32 # Metadata for auto-discovery 

33 provider_key = "LLAMACPP" 

34 company_name = "llama.cpp" 

35 is_cloud = False # Local provider 

36 # Egress exposure (ADR-0007): local inference sink — data stays on the box. 

37 egress_exposure = Exposure.CONTAINED 

38 

39 @classmethod 

40 def create_llm(cls, model_name=None, temperature=0.7, **kwargs): 

41 """Create a ChatOpenAI client pointed at llama-server.""" 

42 from ....config.thread_settings import get_setting_from_snapshot 

43 

44 settings_snapshot = kwargs.get("settings_snapshot") 

45 

46 url = get_setting_from_snapshot( 

47 "llm.llamacpp.url", 

48 cls.default_base_url, 

49 settings_snapshot=settings_snapshot, 

50 ) 

51 

52 kwargs["base_url"] = normalize_url(url) 

53 # Real key when configured (llama-server behind an auth proxy), 

54 # otherwise the unified placeholder; a no-auth llama-server 

55 # ignores it. 

56 kwargs["api_key"] = cls.resolve_api_key_or_placeholder( 

57 settings_snapshot 

58 ) # gitleaks:allow 

59 

60 return super()._create_llm_instance(model_name, temperature, **kwargs) 

61 

62 @classmethod 

63 def is_available(cls, settings_snapshot=None): 

64 """Check whether llama-server is reachable. 

65 

66 Sends ``Authorization: Bearer`` when an API key is configured so 

67 llama-server instances behind an auth proxy are correctly detected 

68 as available. Empty key → no auth header → unauthenticated installs 

69 still work. Mirrors the LMStudio pattern at lmstudio.py:_get_auth_headers. 

70 """ 

71 try: 

72 from ....config.thread_settings import get_setting_from_snapshot 

73 from ....security import safe_get 

74 

75 url = get_setting_from_snapshot( 

76 "llm.llamacpp.url", 

77 cls.default_base_url, 

78 settings_snapshot=settings_snapshot, 

79 ) 

80 base_url = normalize_url(url) 

81 response = safe_get( 

82 f"{base_url}/models", 

83 timeout=1, 

84 headers=cls.build_bearer_header( 

85 settings_snapshot=settings_snapshot 

86 ), 

87 allow_localhost=True, 

88 allow_private_ips=True, 

89 ) 

90 return response.status_code == 200 

91 except Exception: 

92 return False 

93 

94 @classmethod 

95 def requires_auth_for_models(cls): 

96 """llama-server doesn't require authentication for listing models.""" 

97 return False