"""Repeating an LLM's transport calls while its provider fails transiently."""
import logging
import time
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from typing import TypeVar
from openai.types.chat.completion_create_params import ResponseFormat
from eval_framework.contract import ResultCandidate
from eval_framework.llm.base import BaseLLM
from eval_framework.shared.types import RawCompletion, RawLoglikelihood
from template_formatting.formatter import Message
logger = logging.getLogger(__name__)
T = TypeVar("T")
[docs]
class LLMUnavailable(Exception):
"""The model kept failing transiently until the retries ran out, the last failure as ``__cause__``."""
[docs]
@dataclass(frozen=True)
class RetryPolicy:
"""How often and how patiently one call is repeated before the model is given up on.
The pause before a retry doubles from ``backoff_s`` and is capped at ``max_backoff_s``.
"""
# 8, 16, 32, 60 s: about two minutes in all. A throttled OpenRouter key stays throttled for
# minutes while its rejections return in milliseconds, so a budget of a few seconds burned
# through every remaining row of a run within a minute of the throttle setting in.
attempts: int = 5
backoff_s: float = 8.0
max_backoff_s: float = 60.0
[docs]
def pause_before(self, attempt: int) -> float:
return min(self.backoff_s * 2 ** (attempt - 2), self.max_backoff_s)
[docs]
class RetryingLLM(BaseLLM):
"""A model whose generation and logprob calls are repeated while its ``is_retryable`` allows."""
def __init__(self, inner: BaseLLM, policy: RetryPolicy) -> None:
self._inner = inner
self._policy = policy
@property
def name(self) -> str:
return self._inner.name
[docs]
def is_retryable(self, exc: Exception) -> bool:
return self._inner.is_retryable(exc)
[docs]
def with_retry(self, policy: RetryPolicy | None = None) -> BaseLLM:
return self
[docs]
def generate_from_messages(
self,
messages: list[Sequence[Message]],
stop_sequences: list[str] | None = None,
max_tokens: int | None = None,
temperature: float | None = None,
top_p: float | None = None,
response_format: ResponseFormat | None = None,
) -> list[RawCompletion]:
return self._repeat(
lambda: self._inner.generate_from_messages(
messages, stop_sequences, max_tokens, temperature, top_p, response_format
)
)
[docs]
def generate_from_samples(
self,
samples: list[ResultCandidate],
stop_sequences: list[str] | None = None,
max_tokens: int | None = None,
temperature: float | None = None,
top_p: float | None = None,
response_format: ResponseFormat | None = None,
) -> list[RawCompletion]:
return self._repeat(
lambda: self._inner.generate_from_samples(
samples, stop_sequences, max_tokens, temperature, top_p, response_format
)
)
[docs]
def logprobs(self, samples: list[ResultCandidate]) -> list[RawLoglikelihood]:
return self._repeat(lambda: self._inner.logprobs(samples))
[docs]
def post_process_completion(self, completion: str, sample: ResultCandidate) -> str:
return self._inner.post_process_completion(completion, sample)
def _repeat(self, call: Callable[[], T]) -> T:
for attempt in range(1, self._policy.attempts + 1):
try:
return call()
except Exception as exc:
if not self._inner.is_retryable(exc):
raise
if attempt == self._policy.attempts:
raise LLMUnavailable(
f"{self.name} failed {attempt} times; last failure: {exc.__class__.__name__}: {exc}"
) from exc
pause_s = self._policy.pause_before(attempt + 1)
logger.warning(
"%s call failed (attempt %d/%d): %s: %s; retrying in %.0f s",
self.name,
attempt,
self._policy.attempts,
exc.__class__.__name__,
exc,
pause_s,
)
time.sleep(pause_s)
raise AssertionError("unreachable: the retry loop returns or raises")