Source code for eval_framework.tasks.task_style

"""Task-style helpers and strategy classes for choice-based evaluation tasks.

This module provides injectable styling strategies — ``MCStyle``, ``ClozeStyle``, ``MCCompletionStyle``
and ``BPBStyle`` — that render one choice item (question plus answer options) into the prompt text,
answer cue, scored candidates and ground truth. The styler also declares the response type, metrics
and metadata of its style.

Quick-start
-----------
A choice-based benchmark pairs a ``ChoiceReader`` (item -> raw question, choices, correct index) with
a styler via ``ComposedBenchmark.choice``, which drives both the scored sample and the few-shot
demonstrations from the same pair:

.. code-block:: python

    benchmark = ComposedBenchmark.choice(
        id="MyBenchmark",
        reader=MyChoiceReader(),
        styler=ClozeStyle(question_prefix="Question: ", cue_text="Answer:"),
        sample_split="test",
        fewshot_split="train",
        dataset_policy=Pinned("my/dataset"),
    )

Task families with MC, Cloze and BPB variants share the reader and differ only in the styler each
variant's builder injects.
"""

import hashlib
import random
from abc import ABC, abstractmethod
from collections.abc import Callable
from enum import Enum
from typing import TYPE_CHECKING, Any, Self, final

from eval_framework.contract import ResponseType
from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
    AccuracyBayesianLoglikelihood,
    AccuracyLoglikelihood,
    AccuracyNormLoglikelihood,
)
from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
from eval_framework.metrics.loglikelihood.bpb_variants import BitsPerByteVariantsLoglikelihood
from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import ConfidenceWeightedAccuracy
from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
from eval_framework.metrics.loglikelihood.ternary import TernaryScore
from eval_framework.tasks.utils import get_n_letters

if TYPE_CHECKING:
    from eval_framework.metrics.base import BaseMetric


[docs] class TaskStyle(Enum): MULTIPLE_CHOICE = "multiple_choice" CLOZE = "cloze" BPB = "bpb"
[docs] class Language(Enum): """A language the stylers have default question/answer wording for. Members are named by ISO 639-3 code and valued by English name """ CES = "Czech" DAN = "Danish" DEU = "German" ENG = "English" FIN = "Finnish" FRA = "French" HUN = "Hungarian" ITA = "Italian" NLD = "Dutch" POL = "Polish" POR = "Portuguese" RUS = "Russian" SPA = "Spanish" UKR = "Ukrainian"
# A subject-templated preamble: maps a subject label to the preamble text (e.g. MMLU's # "... about {subject}."). Fixed preambles simply ignore the argument. InitialPrompt = Callable[[str], str] # Default (question_prefix, cue_text) per language; extend for new languages as needed. _DEFAULT_QUESTION_CUE_TEXT: dict[Language, tuple[str, str]] = { Language.ENG: ("Question: ", "Answer:"), Language.DEU: ("Frage: ", "Antwort:"), Language.FRA: ("Question : ", "Réponse :"), Language.SPA: ("Pregunta: ", "Respuesta:"), Language.ITA: ("Domanda: ", "Risposta:"), Language.CES: ("Otázka: ", "Odpověď:"), Language.DAN: ("Spørgsmål: ", "Svar:"), Language.FIN: ("Kysymys: ", "Vastaus:"), Language.NLD: ("Vraag: ", "Antwoord:"), Language.POL: ("Pytanie: ", "Odpowiedź:"), Language.RUS: ("Вопрос: ", "Ответ:"), Language.UKR: ("Питання: ", "Відповідь:"), Language.HUN: ("Kérdés: ", "Válasz:"), Language.POR: ("Pergunta: ", "Resposta:"), }
[docs] def question_cue_text(language: Language) -> tuple[str, str]: if language not in _DEFAULT_QUESTION_CUE_TEXT: known = ", ".join(lang.name for lang in _DEFAULT_QUESTION_CUE_TEXT) raise ValueError(f"No default question prefix and cue for {language.name}; known languages: {known}.") return _DEFAULT_QUESTION_CUE_TEXT[language]
# --------------------------------------------------------------------------- # Task styler strategy classes # ---------------------------------------------------------------------------
[docs] class TaskStyler(ABC): """Strategy object that controls prompt assembly and scoring for choice-based tasks. Concrete implementations (``MCStyle``, ``ClozeStyle``) are injected into the ``Choice`` eval kind alongside a ``ChoiceReader``, so benchmark authors only supply data access. Attributes: response_type: The response type the task should use (e.g. LOGLIKELIHOODS). metrics: Default metric classes for tasks using this styler. task_style: Discriminator for metadata (MULTIPLE_CHOICE or CLOZE). question_prefix: String prepended to the raw question. """ response_type: ResponseType metrics: list[type["BaseMetric"]] task_style: TaskStyle question_prefix: str
[docs] @abstractmethod def get_instruction_text(self, raw_question: str, choices: list[str]) -> str: """Build the instruction/prompt text from a question and answer choices."""
[docs] @abstractmethod def get_ground_truth(self, choices: list[str], correct_index: int) -> str: """Return the ground-truth string for scoring."""
[docs] @abstractmethod def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str] | None: """Return completion candidates for scoring, or ``None`` for free generation tasks. ``correct_index`` is only required by ``BPBStyle``, which scores solely the ground-truth completion. ``MCStyle`` and ``ClozeStyle`` score all choices and ignore it; callers may omit it when using those stylers. """
[docs] @abstractmethod def get_cue_text(self) -> str: """Return the assistant cue appended after the prompt (e.g. ``"Answer:"``)."""
[docs] def get_question_text(self, raw_question: str) -> str: """Build the full question line (prefix + raw question). Override in a subclass for non-standard question formats (e.g. HellaSwag's ``"activity: context"`` form). """ return f"{self.question_prefix}{raw_question}"
[docs] def get_fewshot_target_text(self, choices: list[str], correct_index: int) -> str: """Return the few-shot target string (cue + ground truth).""" return f"{self.get_cue_text()}{self.get_ground_truth(choices, correct_index)}"
[docs] def get_extra_metadata(self) -> dict: """Return styler-specific metadata to merge into the task's metadata.""" return {"task_style": self.task_style.value}
[docs] def initial_prompt(self, subject_label: str) -> str | None: """A preamble prepended once at the top of the prompt for the given subject, or None (the default: no preamble). Subject-templated preambles (e.g. MMLU's "... about {subject}.") read ``subject_label``; fixed preambles ignore it.""" return None
[docs] @final def with_abstention_option(self, abstention_option: str) -> "IdkStyle": """Wrap this styler so the model may also abstain with ``abstention_option`` (scored alongside the real answers, with confidence-aware metrics). See ``IdkStyle``.""" return IdkStyle(inner=self, abstention_option=abstention_option)
[docs] @classmethod def for_language(cls, language: Language, **kwargs: Any) -> Self: """Factory that fills ``question_prefix`` and ``cue_text`` from language defaults. Any explicitly passed keyword arguments take precedence over the defaults. """ if language in _DEFAULT_QUESTION_CUE_TEXT: prefix, cue = question_cue_text(language) kwargs.setdefault("question_prefix", prefix) kwargs.setdefault("cue_text", cue) return cls(**kwargs)
[docs] class MCStyle(TaskStyler): """Multiple-choice styler: choices shown in prompt, model scored over letter labels. Args: question_prefix: Prepended to the raw question (default ``"Question: "``). cue_text: Assistant cue after the prompt (default ``"Answer:"``). space_prefixed_labels: When ``True``, each option line starts with a space (``" A. choice"`` — OLMES-style). Default ``False``. initial_prompt: Optional subject-templated preamble prepended once at the top of the prompt (e.g. MMLU's "... about {subject}."). Default ``None`` (no preamble). Assembled prompt example (default settings, 3 choices):: "Question: What is the capital of France?\\nA. Berlin\\nB. Paris\\nC. London\\n" Scored completions: [" A", " B", " C"] Ground truth: " B" """ response_type = ResponseType.LOGLIKELIHOODS metrics: list[type["BaseMetric"]] = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, BitsPerByteVariantsLoglikelihood, ] task_style = TaskStyle.MULTIPLE_CHOICE def __init__( self, question_prefix: str = "Question: ", cue_text: str = "Answer:", space_prefixed_labels: bool = False, initial_prompt: InitialPrompt | None = None, ) -> None: self.question_prefix = question_prefix self._cue_text = cue_text self.space_prefixed_labels = space_prefixed_labels self._initial_prompt = initial_prompt
[docs] def initial_prompt(self, subject_label: str) -> str | None: return self._initial_prompt(subject_label) if self._initial_prompt is not None else None
[docs] def get_cue_text(self) -> str: return self._cue_text
[docs] def get_instruction_text(self, raw_question: str, choices: list[str]) -> str: return format_mc_prompt( self.get_question_text(raw_question), choices, space_prefixed_labels=self.space_prefixed_labels, )
[docs] def get_ground_truth(self, choices: list[str], correct_index: int) -> str: labels = get_n_letters(len(choices)) return f" {labels[correct_index]}"
[docs] def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]: """Note: `correct_index` is ignored for `MCStyle` and only used for `BPBStyle`.""" return [f" {label}" for label in get_n_letters(len(choices))]
[docs] class MCCompletionStyle(TaskStyler): """Multiple-choice prompt style with generative completion output. Prompt formatting is identical to ``MCStyle`` (choices shown in prompt), but ground truth is the full answer text (for example ``"Paris"`` instead of ``"D"``). Responses are generated via ``ResponseType.COMPLETION`` and scored with ``AccuracyCompletion``. This style is useful for instruction-following or CoT prompts where the model is expected to generate an answer text rather than be scored via loglikelihood over fixed label candidates. Args: question_prefix: Prepended to the raw question (default ``"Question: "``). cue_text: Assistant cue after the prompt (default ``"Answer:"``). space_prefixed_labels: When ``True``, each option line starts with a space (``" A. choice"`` — OLMES-style). Default ``False``. Assembled prompt example (default settings, 4 choices):: "Question: What is the capital of France?\\nA. Munich\\nB. Stuttgart\\nC. Berlin\\nD. Paris\\nAnswer:" Ground truth: "Paris" """ response_type = ResponseType.COMPLETION metrics: list[type["BaseMetric"]] = [AccuracyCompletion] task_style = TaskStyle.MULTIPLE_CHOICE def __init__( self, question_prefix: str = "Question: ", cue_text: str = "Answer:", space_prefixed_labels: bool = False, ) -> None: self.question_prefix = question_prefix self._cue_text = cue_text self.space_prefixed_labels = space_prefixed_labels
[docs] def get_cue_text(self) -> str: return self._cue_text
[docs] def get_instruction_text(self, raw_question: str, choices: list[str]) -> str: return format_mc_prompt( self.get_question_text(raw_question), choices, space_prefixed_labels=self.space_prefixed_labels, )
[docs] def get_ground_truth(self, choices: list[str], correct_index: int) -> str: return choices[correct_index]
[docs] def get_fewshot_target_text(self, choices: list[str], correct_index: int) -> str: """Override to insert a space between cue and answer text.""" return f"{self.get_cue_text()} {self.get_ground_truth(choices, correct_index)}"
[docs] def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str] | None: """Completion-mode tasks do not provide fixed candidate completions.""" return None
[docs] class ClozeStyle(TaskStyler): """Cloze styler: no choices in prompt, model scored over full choice text. Also known as "ranked classification" (RC). The prompt only shows the question; the model's score for each full answer text determines the prediction. Args: question_prefix: Prepended to the raw question (default ``"Question: "``). cue_text: Assistant cue after the prompt (default ``"Answer:"``). trailing_newline: When ``True`` (default), the instruction ends with ``"\\n"``. Set to ``False`` for sentence-completion tasks where the model should continue a fragment directly. initial_prompt: Optional subject-templated preamble prepended once at the top of the prompt. Default ``None`` (no preamble). Assembled prompt example (3 choices):: "Question: What is the capital of France?\\n" Scored completions: [" Berlin", " Paris", " London"] Ground truth: " Paris" Sentence-completion example (trailing_newline=False, cue_text=""):: "The cat sat on the" Scored completions: [" mat", " floor", " sofa"] Ground truth: " mat" """ response_type = ResponseType.LOGLIKELIHOODS metrics: list[type["BaseMetric"]] = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, BitsPerByteVariantsLoglikelihood, ] task_style = TaskStyle.CLOZE def __init__( self, question_prefix: str = "Question: ", cue_text: str = "Answer:", trailing_newline: bool = True, leading_space_continuations: bool = True, initial_prompt: InitialPrompt | None = None, ) -> None: self.question_prefix = question_prefix self._cue_text = cue_text self.trailing_newline = trailing_newline self.leading_space_continuations = leading_space_continuations self._initial_prompt = initial_prompt
[docs] def initial_prompt(self, subject_label: str) -> str | None: return self._initial_prompt(subject_label) if self._initial_prompt is not None else None
[docs] def get_cue_text(self) -> str: return self._cue_text
[docs] def get_instruction_text(self, raw_question: str, choices: list[str]) -> str: text = self.get_question_text(raw_question) return f"{text}\n" if self.trailing_newline else text
[docs] def get_ground_truth(self, choices: list[str], correct_index: int) -> str: return f" {choices[correct_index]}" if self.leading_space_continuations else choices[correct_index]
[docs] def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]: return [f" {c}" for c in choices] if self.leading_space_continuations else [f"{c}" for c in choices]
[docs] class BPBStyle(ClozeStyle): """BPB-only styler: prompt identical to ClozeStyle, but scores only the ground-truth completion. One LLM forward pass per sample instead of N (one per choice), making evaluation significantly faster when accuracy metrics are not needed. Args: question_prefix: Prepended to the raw question (default ``"Question: "``). cue_text: Assistant cue after the prompt (default ``"Answer:"``). trailing_newline: When ``True`` (default), the instruction ends with ``"\\n"``. Assembled prompt example (3 choices):: "Question: What is the capital of France?\\n" Scored completions: [" Paris"] ← ground truth only, one forward pass Ground truth: " Paris" """ metrics: list[type["BaseMetric"]] = [BitsPerByteLoglikelihood, BitsPerByteVariantsLoglikelihood] task_style = TaskStyle.BPB
[docs] def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]: if correct_index is None: raise ValueError( "BPBStyle evaluates the loglikelihood of the ground truth answer only," "and thus requires the correct index." ) return [f" {choices[correct_index]}"] if self.leading_space_continuations else [choices[correct_index]]
[docs] class IdkStyle(TaskStyler): """Adds an explicit abstention option (e.g. "I do not know", "?") to a base styler's candidates. Use this when a benchmark rewards calibrated abstention over confident guessing: it scores the real answers together with the abstention option and reports confidence-aware metrics. Everything else — the prompt formatting and the abstention preamble (via the inner styler's ``initial_prompt``) — is the inner styler's; wrap any base styler, so both multiple-choice (abstain with an extra letter) and cloze (abstain with an extra full answer) tasks are supported. Args: inner: The base styler doing the prompt formatting, preamble, and real-answer scoring. abstention_option: The abstention completion scored alongside the real choices, e.g. ``" I do not know"`` (cloze) or ``" ?"`` (multiple-choice). """ response_type = ResponseType.LOGLIKELIHOODS metrics: list[type["BaseMetric"]] = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, ] def __init__(self, inner: TaskStyler, abstention_option: str) -> None: self._inner = inner self._abstention_option = abstention_option self.task_style = inner.task_style # surfaced in metadata
[docs] def get_instruction_text(self, raw_question: str, choices: list[str]) -> str: return self._inner.get_instruction_text(raw_question, choices)
[docs] def get_ground_truth(self, choices: list[str], correct_index: int) -> str: return self._inner.get_ground_truth(choices, correct_index)
[docs] def get_cue_text(self) -> str: return self._inner.get_cue_text()
[docs] def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]: completions = self._inner.get_possible_completions(choices, correct_index) assert completions is not None # abstention wraps a styler that scores a candidate list return completions + [self._abstention_option]
[docs] def initial_prompt(self, subject_label: str) -> str | None: return self._inner.initial_prompt(subject_label)
# --------------------------------------------------------------------------- # Helper functions # ---------------------------------------------------------------------------
[docs] def shuffle_correct_with_distractors( correct: str, distractors: list[str], seed_text: str, ) -> tuple[list[str], int]: """Deterministically shuffle distractors + correct answer; return (choices, correct_index). Calling this multiple times with the same arguments always returns the same result. Args: correct: The correct answer string. distractors: The distractor strings, e.g. wrong answer choices. seed_text: Text used as the shuffle seed (typically question + answer). Returns: A tuple ``(shuffled_choices, correct_index)`` where ``correct_index`` is the 0-based position of ``correct`` in ``shuffled_choices``. """ choices = [*distractors, correct] seed = int(hashlib.sha256(seed_text.encode()).hexdigest(), 16) rng = random.Random(seed) order = list(range(len(choices))) rng.shuffle(order) shuffled = [choices[i] for i in order] correct_index = order.index(len(choices) - 1) return shuffled, correct_index
[docs] def answer_key_to_index(key: str) -> int: """Convert a letter or 1-based integer answer key to a 0-based index. Datasets sometimes encode the correct answer as a letter ("A", "B", ...) and sometimes as a 1-based integer string ("1", "2", ...). This function normalises both to a 0-based index so task code doesn't need to branch. Args: key: A single-character string. Either a letter ("A"-"Z") or a digit ("1"-"9"). Returns: 0-based index: "A" or "1" → 0, "B" or "2" → 1, etc. """ if key.isdigit(): return int(key) - 1 # Shift 1-based integer by 1. return ord(key.upper()) - ord("A") # Turn letter to 0-based index.
[docs] def format_mc_prompt( question_text: str, choices: list[str], *, space_prefixed_labels: bool = False, ) -> str: """Helper function to format a question and its labeled choices into a multiple-choice prompt. The choices are labeled A, B, C, ... in order and ends with a newline. Args: question_text: The full question string (prefix already included). choices: Ordered list of answer option strings. space_prefixed_labels: When ``True``, each option line is prefixed with a space — " A. choice" instead of "A. choice". This is, e.g., the OLMES-style prompt format. Returns: A string of the form ``"<question_text>\\n[pfx]A. choice0\\n[pfx]B. choice1\\n"``. Examples:: >>> format_mc_prompt("Question: What is 1+1?", ["1", "2", "3"]) 'Question: What is 1+1?\\nA. 1\\nB. 2\\nC. 3\\n' >>> format_mc_prompt("Question: What is 1+1?", ["1", "2"], space_prefixed_labels=True) 'Question: What is 1+1?\\n A. 1\\n B. 2\\n' """ labels = get_n_letters(len(choices)) pfx = " " if space_prefixed_labels else "" options = "\n".join(f"{pfx}{label}. {choice}" for label, choice in zip(labels, choices)) return f"{question_text}\n{options}\n"