Coverage for src/local_deep_research/llm/providers/implementations/llamacpp.py: 100%

1"""llama.cpp LLM provider for Local Deep Research. 

2 

3Talks to llama.cpp's OpenAI-compatible HTTP server (`llama-server`) instead 

4of loading models in-process via `llama-cpp-python`. Modeled after 

5`LMStudioProvider`. For setups that need API key auth or non-default URLs 

6beyond a single endpoint, use the `openai_endpoint` provider directly. 

7""" 

8 

9from ....config.constants import DEFAULT_LLAMACPP_URL 

10from ....settings.exceptions import SettingsValueError 

11from ....utilities.url_utils import normalize_url 

12from ..base import Exposure 

13from ..openai_base import OpenAICompatibleProvider 

14 

15 

16class LlamaCppProvider(OpenAICompatibleProvider): 

17 """llama.cpp provider using its OpenAI-compatible HTTP endpoint. 

18 

19 Run `llama-server -m <model.gguf>` (port 8080 by default) and point 

20 `llm.llamacpp.url` at the server's `/v1` endpoint. 

21 """ 

22 

23 provider_name = "llama.cpp" 

24 # llama-server HAS an API key concept (for setups behind an auth proxy); 

25 # api_key_optional makes the base resolver fall back to a placeholder 

26 # when no key is set, instead of raising. 

27 api_key_setting = "llm.llamacpp.api_key" 

28 api_key_optional = True 

29 url_setting = "llm.llamacpp.url" # type: ignore[assignment] 

30 default_base_url = DEFAULT_LLAMACPP_URL 

31 default_model = "" # User must specify the model loaded by llama-server 

32 

33 # Metadata for auto-discovery 

34 provider_key = "LLAMACPP" 

35 company_name = "llama.cpp" 

36 is_cloud = False # Local provider 

37 # Egress exposure (ADR-0007): local inference sink — data stays on the box. 

38 egress_exposure = Exposure.CONTAINED 

39 

40 @classmethod 

41 def create_llm(cls, model_name=None, temperature=0.7, **kwargs): 

42 """Create a ChatOpenAI client pointed at llama-server.""" 

43 from ....config.thread_settings import get_setting_from_snapshot 

44 

45 settings_snapshot = kwargs.get("settings_snapshot") 

46 

47 url = get_setting_from_snapshot( 

48 "llm.llamacpp.url", 

49 cls.default_base_url, 

50 settings_snapshot=settings_snapshot, 

51 ) 

52 

53 kwargs["base_url"] = normalize_url(url) 

54 # Real key when configured (llama-server behind an auth proxy), 

55 # otherwise the unified placeholder; a no-auth llama-server 

56 # ignores it. 

57 kwargs["api_key"] = cls.resolve_api_key_or_placeholder( 

58 settings_snapshot 

59 ) # gitleaks:allow 

60 

61 return super()._create_llm_instance(model_name, temperature, **kwargs) 

62 

63 @classmethod 

64 def is_available(cls, settings_snapshot=None): 

65 """Check whether llama-server is reachable. 

66 

67 Sends ``Authorization: Bearer`` when an API key is configured so 

68 llama-server instances behind an auth proxy are correctly detected 

69 as available. Empty key → no auth header → unauthenticated installs 

70 still work. Mirrors the LMStudio pattern at lmstudio.py:_get_auth_headers. 

71 """ 

72 try: 

73 from ....config.thread_settings import get_setting_from_snapshot 

74 from ....security import safe_get 

75 

76 url = get_setting_from_snapshot( 

77 "llm.llamacpp.url", 

78 cls.default_base_url, 

79 settings_snapshot=settings_snapshot, 

80 ) 

81 base_url = normalize_url(url) 

82 response = safe_get( 

83 f"{base_url}/models", 

84 timeout=1, 

85 headers=cls.build_bearer_header( 

86 settings_snapshot=settings_snapshot 

87 ), 

88 allow_localhost=True, 

89 allow_private_ips=True, 

90 ) 

91 return response.status_code == 200 

92 except SettingsValueError: 

93 raise 

94 except Exception: 

95 return False 

96 

97 @classmethod 

98 def requires_auth_for_models(cls): 

99 """llama-server doesn't require authentication for listing models.""" 

100 return False