Source code for eval_framework.llm.base

from __future__ import annotations

from abc import ABC, abstractmethod
from collections.abc import Sequence
from typing import TYPE_CHECKING

from openai.types.chat.completion_create_params import ResponseFormat

from eval_framework.contract import ResultCandidate
from eval_framework.shared.types import RawCompletion, RawLoglikelihood
from template_formatting.formatter import Message

if TYPE_CHECKING:
    from eval_framework.llm.retry import RetryPolicy


[docs] class BaseLLM(ABC): @property def name(self) -> str: """ This property is used to name the results folder and identify the eval results. Overwrite this property in the subclass with e.g. the checkpoint name/huggingface model name.""" return self.__class__.__name__
[docs] @abstractmethod def generate_from_messages( self, messages: list[Sequence[Message]], stop_sequences: list[str] | None = None, max_tokens: int | None = None, temperature: float | None = None, top_p: float | None = None, response_format: ResponseFormat | None = None, ) -> list[RawCompletion]: """ stop_sequences and max_tokens are injected by the task if exist. They should be overwritten or extended with the properties of the model. This includes but is not limited to the stop tokens by the evaluated checkpoint (e.g. <|eot_id|> for an instruction finetuned Llama3.1, <|endoftext|> for a pretrained Llama3.1). This function is expected to raise errors which are caught and reported when running the eval. Please also make sure to raise an error in case of sequence length issues. We expect to always raise an error if something impedes the expected completion of a task. Important! The completion is expected to be detokenized and to NOT contain special tokens. Backends must also cut the completion at the first occurrence of a stop sequence and return the text before it; the stop sequence itself must not be part of the completion. Answer extraction relies on this and does no stripping of its own. Returns: List[RawCompletion] """ raise NotImplementedError
[docs] def generate_from_samples( self, samples: list[ResultCandidate], stop_sequences: list[str] | None = None, max_tokens: int | None = None, temperature: float | None = None, top_p: float | None = None, response_format: ResponseFormat | None = None, ) -> list[RawCompletion]: """ stop_sequences and max_tokens are injected by the task if exist. They should be overwritten or extended with the properties of the model. This includes but is not limited to the stop tokens by the evaluated checkpoint (e.g. <|eot_id|> for an instruction finetuned Llama3.1, <|endoftext|> for a pretrained Llama3.1). This function is expected to raise errors which are caught and reported when running the eval. Please also make sure to raise an error in case of sequence length issues. We expect to always raise an error if something impedes the expected completion of a task. Important! The completion is expected to be detokenized and to NOT contain special tokens. Backends must also cut the completion at the first occurrence of a stop sequence and return the text before it; the stop sequence itself must not be part of the completion. Answer extraction relies on this and does no stripping of its own. Returns: List[RawCompletion] """ raise NotImplementedError
[docs] @abstractmethod def logprobs(self, samples: list[ResultCandidate]) -> list[RawLoglikelihood]: """ This function is expected to raise errors which are caught and reported when running the eval. Please also make sure to raise an error in case of sequence length issues. We expect to always raise an error if something prevents the expected completion of a task. """ raise NotImplementedError
[docs] def generate( self, samples: list[ResultCandidate], stop_sequences: list[str] | None = None, max_tokens: int | None = None, temperature: float | None = None, top_p: float | None = None, response_format: ResponseFormat | None = None, ) -> list[RawCompletion]: """Generates a model response for each sample. Uses 'generate_from_samples' to generate responses if implemented, otherwise falls back to 'generate_from_messages'. """ try: return self.generate_from_samples(samples, stop_sequences, max_tokens, temperature, top_p, response_format) except NotImplementedError: messages: list[Sequence[Message]] = [sample.messages for sample in samples] return self.generate_from_messages( messages, stop_sequences, max_tokens, temperature, top_p, response_format )
[docs] def is_retryable(self, exc: Exception) -> bool: """Whether ``with_retry()`` repeats a call that raised ``exc``: by default any failure at all.""" # Ask, don't tell: the wrapper queries the model instead of the model raising a typed error. # Deliberate, so that no adapter has to be rewritten around the retry mechanism to work with it. return not isinstance(exc, NotImplementedError)
[docs] def with_retry(self, policy: RetryPolicy | None = None) -> BaseLLM: """This model with every generation and logprob call repeated while `is_retryable` allows.""" from eval_framework.llm.retry import RetryingLLM, RetryPolicy return RetryingLLM(self, policy or RetryPolicy())
[docs] def post_process_completion(self, completion: str, sample: ResultCandidate) -> str: """ Model-specific post-processing of generated completions. Override this method to apply model-specific cleanup or transformations (e.g., removing specific artifacts such as reasoning traces, handling special tokens). Args: completion: The raw completion string from the model sample: The sample that was used to generate the completion Returns: The post-processed completion string """ return completion
def __del__(self) -> None: """ Method for custom resource cleanup (particularly GPUs) """ pass