Source code for eval_framework.contract
from abc import ABC, abstractmethod
from collections.abc import Iterator, Sequence
from enum import Enum
from typing import TYPE_CHECKING
from pydantic import BaseModel, ConfigDict
from eval_framework.shared.types import BaseMetricContext, Completion
from template_formatting.formatter import BaseFormatter, Message
if TYPE_CHECKING:
from eval_framework.llm.base import BaseLLM
from eval_framework.metrics.base import BaseMetric
[docs]
class ResponseType(Enum):
COMPLETION = "completion"
LOGLIKELIHOODS = "loglikelihoods"
[docs]
class ResultCandidate(BaseModel):
model_config = ConfigDict(extra="forbid", use_attribute_docstrings=True)
sample_id: int
"""This ID denotes the input problem identifier. Aggregators can work with this to find unique problems."""
result_id: int
"""This ID relates to the output result identifier. This subsumes repeats."""
subject: str
messages: list[Message]
ground_truth: str | list[str] | None
possible_completions: list[str] | None
context: BaseMetricContext | list[BaseMetricContext] | None = None
[docs]
class Eval(ABC):
"""The contract a caller relies on to run an evaluation"""
[docs]
@abstractmethod
def iterate_result_candidates(self, num_samples: int | None, *, repeats: int) -> Iterator[ResultCandidate]:
"""Yield ``repeats`` result candidates per sample, consecutively, across all subjects. ``num_samples``
caps samples (not candidates) PER SUBJECT (``None`` = no cap), so a benchmark with S subjects yields
up to ``S * num_samples`` samples. ``sample_id`` is the sample's index within its subject, restarting
at 0 for each subject; ``result_id`` tells the repeats apart and is unique within a subject."""
[docs]
@abstractmethod
def generate_completions(
self,
llm: "BaseLLM",
samples: list[ResultCandidate],
stop_sequences: list[str] | None = None,
max_tokens: int | None = None,
) -> list[Completion]:
"""Run ``llm`` over ``samples`` and return their completions."""
[docs]
@abstractmethod
def get_stop_sequences(self) -> list[str]:
"""Stop sequences the eval requests for completion generation."""
[docs]
@abstractmethod
def get_max_tokens(self) -> int | None:
"""Token limit the eval requests for completion generation, or ``None`` for no limit."""
[docs]
@abstractmethod
def get_response_type(self) -> ResponseType: ...
[docs]
@abstractmethod
def display_name(self) -> str:
"""Human-readable display name. Is allowed to have special characters and whitespaces."""
...
[docs]
class Benchmark(ABC):
"""A benchmark is used to provide means of measuring model performance in a domain.
Benchmark act as factories for `Eval`. They bind all the a prior known information and enrich it
with the arguments provided at runtime to create concrete instances of `Eval` which are used to
provide measurements of the models performance.
"""
[docs]
@abstractmethod
def id(self) -> str:
"Uniquely identifies the benchmark"
[docs]
@abstractmethod
def response_type(self) -> ResponseType:
"""The benchmark's response type"""
[docs]
@abstractmethod
def metrics(self) -> list[type["BaseMetric"]]:
"""The benchmark's metrics"""
[docs]
@abstractmethod
def subjects(self, tokens: list[str] | None = None) -> list[str]:
"""Labels of the subjects a run with these ``--task-subjects`` tokens evaluates (no tokens =
all), exactly as they appear in samples and aggregated-result keys. Raises ``ValueError`` for
a token no subject matches, mirroring what ``create`` does with the same tokens."""
[docs]
@abstractmethod
def display_name(self) -> str:
"""Human-readable display name. Is allowed to have special characters and whitespaces."""
[docs]
@abstractmethod
def create(
self,
num_fewshot: int,
custom_subjects: list[str] | None,
custom_hf_revision: str | None,
seed: int | None = None,
) -> Eval: ...
[docs]
@abstractmethod
def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
"""Render the benchmarks's documentation as markdown."""
...