Coverage for src/local_deep_research/llm/providers/_helpers.py: 100%
73 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-22 21:20 +0000
1"""Shared utilities for LLM provider construction.
3These helpers were previously inlined in ``config/llm_config.get_llm``'s
4procedural ``if/elif`` chain (which is now removed). They live here so
5provider-class ``create_llm`` methods are the single source of truth for
6LLM construction logic — no per-provider drift.
7"""
9import math
10from typing import Any, Final
12from ...settings.exceptions import SettingsValueError
15DEFAULT_LOCAL_CONTEXT_WINDOW_SIZE = 8192
16DEFAULT_CLOUD_CONTEXT_WINDOW_SIZE = 128000
17LOCAL_PROVIDERS = ("ollama", "llamacpp", "lmstudio")
18# compute_max_tokens caps llm.max_tokens at this share of a known context
19# window, which leaves the rest of the window for the prompt.
20MAX_TOKENS_CONTEXT_SHARE = 0.8
22# ``llm.request_timeout`` is a *per-network-operation* bound, not a cap on
23# the total wall-clock duration of a response. httpx (which every provider
24# SDK here sits on) has no total-deadline knob: a scalar expands to
25# ``Timeout(connect=X, read=X, write=X, pool=X)`` where ``read`` is the
26# time allowed *between* received bytes. A provider that keeps dripping
27# data therefore keeps the request alive past X. See
28# ``docs/CONFIGURATION.md`` — the setting is documented as an inactivity
29# bound for exactly this reason.
30DEFAULT_REQUEST_TIMEOUT: Final = 1800
31MIN_REQUEST_TIMEOUT: Final = 1
32# Deliberately no ceiling: a long-running local model on modest hardware
33# can legitimately think for hours, and a cap that is generous enough for
34# those is not a meaningful guard against a typo anyway. Non-finite values
35# are still rejected below, which is what actually prevents an unbounded
36# wait; MAX_MAX_RETRIES stays, because retries MULTIPLY this bound.
38# Connecting is not inference: a blackholed IP should fail fast instead of
39# holding a worker for the full inference budget. Matches the OpenAI and
40# Anthropic SDK defaults (``Timeout(connect=5.0, ...)``), which a scalar
41# timeout would otherwise overwrite.
42CONNECT_TIMEOUT_SECONDS: Final = 5.0
44# Limit the number of SDK retries. Each attempt has its own per-operation
45# timeouts, with SDK backoff or server-directed delays between attempts.
46DEFAULT_MAX_RETRIES: Final = 2
47MIN_MAX_RETRIES: Final = 0
48MAX_MAX_RETRIES: Final = 5
50MODEL_DISCOVERY_TIMEOUT_SECONDS: Final = 30
53def get_local_context_window(settings_snapshot=None):
54 """Resolve ``llm.local_context_window_size``, the local providers' window."""
55 from ...config.thread_settings import get_setting_from_snapshot
57 window_size = get_setting_from_snapshot(
58 "llm.local_context_window_size",
59 DEFAULT_LOCAL_CONTEXT_WINDOW_SIZE,
60 settings_snapshot=settings_snapshot,
61 )
62 return (
63 int(window_size)
64 if window_size is not None
65 else DEFAULT_LOCAL_CONTEXT_WINDOW_SIZE
66 )
69def get_context_window_for_provider(provider_type, settings_snapshot=None):
70 """Resolve effective context window size for a provider.
72 Local providers (ollama, llamacpp, lmstudio) use a smaller default to
73 prevent memory issues. Cloud providers respect
74 ``llm.context_window_unrestricted`` and return ``None`` when
75 unrestricted (the provider auto-handles its own context window).
76 """
77 from ...config.thread_settings import get_setting_from_snapshot
79 if provider_type in LOCAL_PROVIDERS:
80 return get_local_context_window(settings_snapshot)
81 use_unrestricted = get_setting_from_snapshot(
82 "llm.context_window_unrestricted",
83 True,
84 settings_snapshot=settings_snapshot,
85 )
86 if use_unrestricted:
87 return None
88 window_size = get_setting_from_snapshot(
89 "llm.context_window_size",
90 DEFAULT_CLOUD_CONTEXT_WINDOW_SIZE,
91 settings_snapshot=settings_snapshot,
92 )
93 return (
94 int(window_size)
95 if window_size is not None
96 else DEFAULT_CLOUD_CONTEXT_WINDOW_SIZE
97 )
100def compute_max_tokens(
101 settings_snapshot=None, context_window_size=None
102) -> int | None:
103 """Resolve effective max_tokens for ``ChatXxx(max_tokens=...)``.
105 Caps at 80% of ``context_window_size`` (when provided) to leave room
106 for the prompt. Returns ``None`` (caller should omit the kwarg) when:
108 - ``llm.supports_max_tokens`` is False, OR
109 - ``llm.max_tokens`` is unset / explicitly None in the snapshot.
111 Omitting the kwarg when the setting is absent matches the pre-refactor
112 live-class behavior (the provider SDK's own default applies); a
113 hardcoded fallback like the dead chain's 100000 exceeds the output
114 limit of most cloud models. Production users have ``llm.max_tokens``
115 populated from ``default_settings.json`` (currently 30000), so the
116 unset branch only fires for partial-snapshot programmatic callers.
118 ``llm.max_tokens`` is read with an explicit ``None`` default (#5984),
119 so an absent key resolves to ``None`` (kwarg omitted) instead of
120 raising ``NoSettingsContextError``. Provider-level ``except
121 NoSettingsContextError`` wrappers around this helper are defensive
122 only.
123 """
124 from ...config.thread_settings import get_setting_from_snapshot
126 if not get_setting_from_snapshot(
127 "llm.supports_max_tokens",
128 True,
129 settings_snapshot=settings_snapshot,
130 ):
131 return None
132 raw = get_setting_from_snapshot(
133 "llm.max_tokens",
134 default=None,
135 settings_snapshot=settings_snapshot,
136 )
137 if raw is None:
138 return None
139 max_tokens = int(raw)
140 if context_window_size is not None:
141 max_tokens = min(
142 max_tokens, int(context_window_size * MAX_TOKENS_CONTEXT_SHARE)
143 )
144 return max_tokens
147def _get_setting_value(settings_snapshot, key: str, default: Any) -> Any:
148 """Read an optional setting; invalid configured values must propagate."""
149 from ...config.thread_settings import (
150 get_setting_from_snapshot,
151 NoSettingsContextError,
152 )
154 try:
155 return get_setting_from_snapshot(
156 key, default, settings_snapshot=settings_snapshot
157 )
158 except NoSettingsContextError:
159 return default
162def _validate_number(
163 key: str,
164 value: Any,
165 *,
166 default: float,
167 minimum: float,
168 maximum: float | None,
169) -> float:
170 """Validate ``value`` as a finite float inside ``[minimum, maximum]``.
172 ``maximum=None`` means no upper bound; the value is still required to be
173 finite, so ``inf`` cannot pass as 'no timeout'.
175 Only an unset value uses ``default``. Invalid configured values raise
176 without including the value or a parser exception in the error.
177 """
178 if value is None:
179 return float(default)
180 if isinstance(value, bool):
181 raise SettingsValueError(
182 key, source="snapshot", reason="expected a finite number"
183 )
184 number = None
185 try:
186 number = float(value)
187 except (TypeError, ValueError, OverflowError):
188 pass
189 # Raise outside the parser handler so its exception cannot expose a
190 # configured value through __context__.
191 if number is None or not math.isfinite(number):
192 raise SettingsValueError(
193 key, source="snapshot", reason="expected a finite number"
194 )
195 if number < minimum:
196 raise SettingsValueError(
197 key, source="snapshot", reason=f"must be at least {minimum}"
198 )
199 if maximum is not None and number > maximum:
200 raise SettingsValueError(
201 key, source="snapshot", reason=f"must be at most {maximum}"
202 )
203 return number
206def _resolve_bounded_number(
207 settings_snapshot,
208 key: str,
209 *,
210 default: float,
211 minimum: float,
212 maximum: float | None,
213) -> float:
214 """Read a numeric setting and validate its supported bounds.
216 ``maximum=None`` means no upper bound; the value is still required to
217 be a finite number, so ``inf`` cannot slip through as 'no timeout'.
218 """
219 return _validate_number(
220 key,
221 _get_setting_value(settings_snapshot, key, default),
222 default=default,
223 minimum=minimum,
224 maximum=maximum,
225 )
228def resolve_request_timeout(settings_snapshot=None) -> float:
229 """Resolve the per-operation request timeout shared by all clients.
231 The returned value bounds each individual network operation (connect,
232 write, and the wait for the *next* response byte) — not the total
233 duration of a response. Always returns a finite number of at least
234 ``MIN_REQUEST_TIMEOUT``; there is no upper bound.
235 """
236 return _resolve_bounded_number(
237 settings_snapshot,
238 "llm.request_timeout",
239 default=DEFAULT_REQUEST_TIMEOUT,
240 minimum=MIN_REQUEST_TIMEOUT,
241 maximum=None,
242 )
245def resolve_max_retries(settings_snapshot=None) -> int:
246 """Resolve the bounded SDK retry count shared by all clients.
248 Retries multiply the number of attempts, so the configured value must
249 not exceed ``MAX_MAX_RETRIES``. Each attempt has per-operation timeouts;
250 SDK backoff and server-directed delays also affect total elapsed time.
251 """
252 return int(
253 _resolve_bounded_number(
254 settings_snapshot,
255 "llm.max_retries",
256 default=DEFAULT_MAX_RETRIES,
257 minimum=MIN_MAX_RETRIES,
258 maximum=MAX_MAX_RETRIES,
259 )
260 )
263def build_timeout(seconds: float) -> tuple[float, float, float, float]:
264 """Expand a scalar bound into a hashable per-operation timeout tuple.
266 Returns ``(connect, read, write, pool)``, the 4-tuple form httpx
267 accepts wherever it accepts a ``Timeout`` object
268 (``httpx._config.Timeout.__init__`` unpacks a tuple in exactly this
269 order). The effective client timeout is identical either way:
270 ``Timeout(connect=min(5, X), read=X, write=X, pool=X)``. The connect
271 slot is bounded *at most* at 5 seconds, not pinned to it — below a
272 5-second ``seconds`` the ``min`` yields ``seconds``, so raising the
273 setting from 2 to 4 does raise the connect bound as well.
275 **For client construction only.** ``openai._models
276 .FinalRequestOptions.timeout`` is pydantic-validated and rejects a
277 tuple, so a per-request ``timeout=`` keyword (e.g.
278 ``client.models.list(timeout=...)``) must use ``build_httpx_timeout``
279 instead. Only the client-level ``timeout=`` is untyped.
281 The openai SDK also stringifies whatever it is given into the
282 informational ``x-stainless-read-timeout`` request header, so the
283 tuple's repr rides along on the clients that take it. The effective
284 bounds are unaffected; the header is not read by this codebase.
286 **The tuple shape is load-bearing, not cosmetic.** ``langchain_openai``
287 builds ``ChatOpenAI``'s httpx clients through an ``@lru_cache``'d
288 factory and silently falls back to a per-instance client when the
289 timeout is unhashable — ``_client_utils._get_default_httpx_client``:
290 *"Uses cached client unless timeout is ``httpx.Timeout``, which is not
291 hashable"*. ``httpx.Timeout`` defines ``__eq__`` without ``__hash__``,
292 so handing a ``Timeout`` object to ``ChatOpenAI(request_timeout=...)``
293 would give every LLM its own connection pool. That would invert the
294 invariants recorded in ``utilities/llm_utils.py`` (``_close_base_llm``
295 deliberately does *not* close ChatOpenAI/ChatAnthropic clients because
296 they are shared) and in ``docs/developing/resource-cleanup.md``. With
297 the tuple those stay true, so neither needs changing.
299 Sharing is also why sync entry points must keep calling the sync
300 ``invoke``: the cached *async* client is bound to the event loop that
301 created it, so a second ``asyncio.run()`` in the same process reuses a
302 client whose loop is closed (issue #6293). That is a property of
303 langchain's cache, not of this bound — an unhashable timeout only
304 masked it by accident.
306 Passing a bare scalar instead would set ``connect`` to the same large
307 value, so a TCP connect to a blackholed address would hold a worker
308 for the whole inference budget. A 2-tuple is not an option either: it
309 leaves ``write`` and ``pool`` at ``None`` (unbounded), so all four
310 slots are always filled.
311 """
312 bounded = float(seconds)
313 return (min(CONNECT_TIMEOUT_SECONDS, bounded), bounded, bounded, bounded)
316def build_httpx_timeout(seconds: float, timeout_cls):
317 """``build_timeout`` as the SDK's own ``Timeout`` object.
319 For every client whose httpx client is per-instance regardless of
320 hashability, which is all of them except ``ChatOpenAI``:
321 ``ollama.Client`` is constructed once per ``ChatOllama``
322 /``OllamaEmbeddings``; ``OpenAIEmbeddings`` never consults
323 ``_get_default_httpx_client`` at all; and the model-discovery clients
324 are fresh SDK instances built per call. Only
325 ``ChatOpenAI`` routes through the ``@lru_cache``'d factory, so only
326 there does the tuple of ``build_timeout`` buy anything.
328 Use the target SDK's exported type — ``anthropic.Timeout``,
329 ``openai.Timeout``, or ``httpx.Timeout`` for Ollama — to match its
330 native transport. This is not interchangeable: Anthropic and Ollama
331 ship ``httpx`` while openai ships ``httpx2``, and the two ``Timeout``
332 classes do not recognise each other. Handing an ``httpx.Timeout`` to
333 ``openai.OpenAI(timeout=...)`` is silently treated as a scalar, which
334 nests it into all four slots of an ``httpx2.Timeout``.
335 """
336 connect, read, write, pool = build_timeout(seconds)
337 return timeout_cls(connect=connect, read=read, write=write, pool=pool)