Source code for eval_framework.benchmarks.mbpp_ellamind

"""Multilingual MBPP (EllaMind): https://huggingface.co/datasets/ellamind/mbpp-multilingual

The multilingual counterpart of MBPP — the dataset mirrors the ``text`` / ``code`` / ``test_list`` schema, so these
reuse the builders and reconstruct functions from the ``mbpp`` module with a per-language instruction wrapper.
Unlike the English variants (fixed English 3-shot block), these sample demonstrations from the eval split of the
language. Each language is one config of the dataset (``deu``).

Variants, per language:

- ``OLMES`` / ``BPB_OLMES``: the OLMES instruction; generation scored by execution, or BPB of the gold solution.
- ``EvalPlus`` / ``BPB_EvalPlus``: the EvalPlus instruction (normalised line endings); likewise.
- ``EvalPlusInstruct``: the EvalPlus instruction without a forced cue; the code block is extracted from the response.

Adding a language: extend ``LanguageCode`` and add its wording to ``_LOCALES``.
"""

from collections.abc import Callable
from dataclasses import dataclass
from typing import Any, Literal, get_args

from eval_framework.answer import ReconstructProgram
from eval_framework.benchmarks.mbpp import (
    END,
    OLMES_STOPS,
    SingleGoldReader,
    bpb,
    evalplus_reconstruct,
    execution,
    instruct_reconstruct,
    olmes_reconstruct,
)
from eval_framework.benchmarks.utils.ellamind_languages import ellamind_language
from eval_framework.contract import Benchmark
from eval_framework.fewshot import ChoiceRenderer, FewShot, SampleSplit
from eval_framework.subjects import ListOfSubjects
from eval_framework.tasks.dataset_loading import DatasetPolicy
from eval_framework.tasks.dataset_revisions import pinned_by_framework
from eval_framework.tasks.task_style import BPBStyle

MBPP_ELLAMIND_DATASET_PATH = "ellamind/mbpp-multilingual"

TaskVariant = Literal["OLMES", "BPB_OLMES", "EvalPlus", "BPB_EvalPlus", "EvalPlusInstruct"]

# The languages this file offers (see ``ellamind_languages`` for the token -> language/config table).
LanguageCode = Literal["DE", "FR"]
_LANGUAGES: tuple[LanguageCode, ...] = get_args(LanguageCode)

# Variant -> task-id template. The ids predate the naming convention and are kept for continuity.
_ID_TEMPLATES: dict[TaskVariant, str] = {
    "OLMES": "MBPP{language}_OLMES",
    "BPB_OLMES": "MBPP{language}_BPB_OLMES",
    "EvalPlus": "MBPP{language}_EvalPlus",
    "BPB_EvalPlus": "MBPP{language}_BPB_EvalPlus",
    "EvalPlusInstruct": "MBPP{language}EvalPlusInstruct",
}


[docs] @dataclass(frozen=True, kw_only=True) class MbppLocale: """One language's wording of the MBPP instruction wrapper and answer cues.""" preamble: str """Opens the instruction, before the fenced problem text and first test.""" preamble_stop: str """The opening words of ``preamble``, as a stop sequence: an EvalPlus generation that starts a new problem is cut there.""" olmes_cue: str """Assistant cue of the OLMES variant; opens the code fence the model continues.""" evalplus_cue: str """Assistant cue of the EvalPlus variants; opens the code fence the model continues.""" def __post_init__(self) -> None: if not self.preamble.startswith(self.preamble_stop.lstrip("\n")): raise ValueError(f"preamble_stop {self.preamble_stop!r} must open the preamble {self.preamble!r}")
_LOCALES: dict[LanguageCode, MbppLocale] = { "DE": MbppLocale( preamble=( "Bitte erstelle ein in sich geschlossenes Python-Skript, " "das das folgende Problem in einem Markdown-Code-Block löst:" ), preamble_stop="\nBitte erstelle", olmes_cue="Hier ist die fertige Funktion:\n\n```python\n", evalplus_cue=( "Hier ist ein Python-Skript mit einer in sich geschlossenen Funktion, " "die das Problem löst und die entsprechenden Tests besteht:\n```python" ), ), "FR": MbppLocale( preamble=( "Veuillez créer un script Python autonome qui résout le problème suivant dans un bloc de code Markdown :" ), preamble_stop="\nVeuillez créer", olmes_cue="Voici la fonction complète :\n\n```python\n", evalplus_cue=( "Voici un script Python avec une fonction autonome qui résout le problème et" " réussit les tests correspondants :\n```python" ), ), } def _text(item: dict[str, Any]) -> str: return item["text"] if "text" in item else item["prompt"] def _norm_text(item: dict[str, Any]) -> str: # EvalPlus additionally normalises Windows line endings in the problem text. return _text(item).replace("\r\n", "\n").strip() def _norm_code(item: dict[str, Any]) -> str: return (item["code"] or "").replace("\r\n", "\n").strip() class _MbppInstruction: """Renders the instruction wrapper of one language around an item's problem text and first test.""" def __init__(self, locale: MbppLocale) -> None: self._preamble = locale.preamble def olmes(self, item: dict[str, Any]) -> str: return f"{self._preamble}\n```\n{_text(item).strip()}\n{item['test_list'][0]}\n```\n" def evalplus(self, item: dict[str, Any]) -> str: return f"{self._preamble}\n```\n{_norm_text(item)}\n{item['test_list'][0]}\n```\n" # A variant builder: ``(id, locale, subjects, dataset) -> Benchmark``. _Builder = Callable[[str, MbppLocale, ListOfSubjects, DatasetPolicy], Benchmark] def _olmes(id: str, locale: MbppLocale, subjects: ListOfSubjects, dataset: DatasetPolicy) -> Benchmark: return execution( id, dataset_path=MBPP_ELLAMIND_DATASET_PATH, instruction=_MbppInstruction(locale).olmes, cue=locale.olmes_cue, fewshot_target=lambda demo: demo["code"] + "\n", answer=ReconstructProgram(olmes_reconstruct, stop_sequences=OLMES_STOPS), fewshot_source=SampleSplit(), # EllaMind ships only a test split; sample (leak-safe) from it subjects=subjects, dataset=dataset, ) def _bpb_olmes(id: str, locale: MbppLocale, subjects: ListOfSubjects, dataset: DatasetPolicy) -> Benchmark: # The BPB_OLMES prompt carries no instruction wrapper, so nothing of the locale is rendered. reader = SingleGoldReader( question=lambda item: _norm_text(item) + "\n```python\n", gold=lambda item: _norm_code(item) + "\n```", ) styler = BPBStyle(question_prefix="", cue_text="", trailing_newline=False, leading_space_continuations=False) return bpb( id, dataset_path=MBPP_ELLAMIND_DATASET_PATH, reader=reader, styler=styler, fewshot=FewShot(SampleSplit(), ChoiceRenderer(reader, styler)), subjects=subjects, dataset=dataset, ) def _evalplus(id: str, locale: MbppLocale, subjects: ListOfSubjects, dataset: DatasetPolicy) -> Benchmark: return execution( id, dataset_path=MBPP_ELLAMIND_DATASET_PATH, instruction=_MbppInstruction(locale).evalplus, cue=locale.evalplus_cue, fewshot_target=lambda demo: locale.evalplus_cue + "\n" + _norm_code(demo) + "\n```", answer=ReconstructProgram(evalplus_reconstruct, stop_sequences=["```", "\nassert", locale.preamble_stop]), fewshot_source=SampleSplit(), subjects=subjects, dataset=dataset, ) def _bpb_evalplus(id: str, locale: MbppLocale, subjects: ListOfSubjects, dataset: DatasetPolicy) -> Benchmark: reader = SingleGoldReader( question=_MbppInstruction(locale).evalplus, gold=lambda item: "\n" + _norm_code(item) + "\n" + END, ) styler = BPBStyle( question_prefix="", cue_text=locale.evalplus_cue, trailing_newline=False, leading_space_continuations=False ) return bpb( id, dataset_path=MBPP_ELLAMIND_DATASET_PATH, reader=reader, styler=styler, fewshot=FewShot(SampleSplit(), ChoiceRenderer(reader, styler)), subjects=subjects, dataset=dataset, ) def _evalplus_instruct(id: str, locale: MbppLocale, subjects: ListOfSubjects, dataset: DatasetPolicy) -> Benchmark: return execution( id, dataset_path=MBPP_ELLAMIND_DATASET_PATH, instruction=_MbppInstruction(locale).evalplus, cue="", # no forced preamble; the code block is extracted from the free-form response fewshot_target=lambda demo: f"```python\n{_norm_code(demo)}\n```", answer=ReconstructProgram(instruct_reconstruct, stop_sequences=[]), fewshot_source=SampleSplit(), subjects=subjects, dataset=dataset, ) _BUILDERS: dict[TaskVariant, _Builder] = { "OLMES": _olmes, "BPB_OLMES": _bpb_olmes, "EvalPlus": _evalplus, "BPB_EvalPlus": _bpb_evalplus, "EvalPlusInstruct": _evalplus_instruct, }
[docs] def mbpp_ellamind(task_variant: TaskVariant, language: LanguageCode, dataset: DatasetPolicy | None = None) -> Benchmark: """Build ``MBPP{language}_{variant}``, e.g. ``mbpp_ellamind("OLMES", "DE")``.""" config = ellamind_language(language).config dataset_policy = dataset if dataset is not None else pinned_by_framework(MBPP_ELLAMIND_DATASET_PATH) return _BUILDERS[task_variant]( _ID_TEMPLATES[task_variant].format(language=language), _LOCALES[language], ListOfSubjects([config]), dataset_policy, )
MBPP_ELLAMIND_BENCHMARKS: list[Benchmark] = [ mbpp_ellamind(task_variant, language) for language in _LANGUAGES for task_variant in get_args(TaskVariant) ]