Source code for eval_framework.contract

from abc import ABC, abstractmethod
from collections.abc import Iterator, Sequence
from enum import Enum
from typing import TYPE_CHECKING

from pydantic import BaseModel, ConfigDict

from eval_framework.shared.types import BaseMetricContext, Completion
from template_formatting.formatter import BaseFormatter, Message

if TYPE_CHECKING:
    from eval_framework.llm.base import BaseLLM
    from eval_framework.metrics.base import BaseMetric


[docs] class ResponseType(Enum): COMPLETION = "completion" LOGLIKELIHOODS = "loglikelihoods"
[docs] class ResultCandidate(BaseModel): model_config = ConfigDict(extra="forbid", use_attribute_docstrings=True) sample_id: int """This ID denotes the input problem identifier. Aggregators can work with this to find unique problems.""" result_id: int """This ID relates to the output result identifier. This subsumes repeats.""" subject: str messages: list[Message] ground_truth: str | list[str] | None possible_completions: list[str] | None context: BaseMetricContext | list[BaseMetricContext] | None = None
[docs] class Eval(ABC): """The contract a caller relies on to run an evaluation"""
[docs] @abstractmethod def iterate_result_candidates(self, num_samples: int | None, *, repeats: int) -> Iterator[ResultCandidate]: """Yield ``repeats`` result candidates per sample, consecutively, across all subjects. ``num_samples`` caps samples (not candidates) PER SUBJECT (``None`` = no cap), so a benchmark with S subjects yields up to ``S * num_samples`` samples. ``sample_id`` is the sample's index within its subject, restarting at 0 for each subject; ``result_id`` tells the repeats apart and is unique within a subject."""
[docs] @abstractmethod def generate_completions( self, llm: "BaseLLM", samples: list[ResultCandidate], stop_sequences: list[str] | None = None, max_tokens: int | None = None, ) -> list[Completion]: """Run ``llm`` over ``samples`` and return their completions."""
[docs] @abstractmethod def get_metadata(self) -> dict[str, str | list[str]]: """Descriptive metadata about the eval for result reporting."""
[docs] @abstractmethod def get_stop_sequences(self) -> list[str]: """Stop sequences the eval requests for completion generation."""
[docs] @abstractmethod def get_max_tokens(self) -> int | None: """Token limit the eval requests for completion generation, or ``None`` for no limit."""
[docs] @abstractmethod def get_response_type(self) -> ResponseType: ...
[docs] @abstractmethod def display_name(self) -> str: """Human-readable display name. Is allowed to have special characters and whitespaces.""" ...
[docs] class Benchmark(ABC): """A benchmark is used to provide means of measuring model performance in a domain. Benchmark act as factories for `Eval`. They bind all the a prior known information and enrich it with the arguments provided at runtime to create concrete instances of `Eval` which are used to provide measurements of the models performance. """
[docs] @abstractmethod def id(self) -> str: "Uniquely identifies the benchmark"
[docs] @abstractmethod def response_type(self) -> ResponseType: """The benchmark's response type"""
[docs] @abstractmethod def metrics(self) -> list[type["BaseMetric"]]: """The benchmark's metrics"""
[docs] @abstractmethod def subjects(self, tokens: list[str] | None = None) -> list[str]: """Labels of the subjects a run with these ``--task-subjects`` tokens evaluates (no tokens = all), exactly as they appear in samples and aggregated-result keys. Raises ``ValueError`` for a token no subject matches, mirroring what ``create`` does with the same tokens."""
[docs] @abstractmethod def display_name(self) -> str: """Human-readable display name. Is allowed to have special characters and whitespaces."""
[docs] @abstractmethod def create(
self, num_fewshot: int, custom_subjects: list[str] | None, custom_hf_revision: str | None, seed: int | None = None, ) -> Eval: ...
[docs] @abstractmethod def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str: """Render the benchmarks's documentation as markdown.""" ...