"""Multilingual Hendrycks Math (EllaMind), minerva-style — composed.
https://huggingface.co/datasets/ellamind/hendrycks-math-multilingual
The per-language counterparts of the Minerva-OLMES MATH tasks, one dataset config per language (``deu`` or
``fra``). Each language sets its problem and solution markers (``Aufgabe:`` / ``Lösung:`` in German), its
final-answer line, its ``minerva_<lang>`` answer extraction and its Minerva metrics. Few-shot demonstrations
are sampled from the dataset (single-paragraph solutions only, so the ``\\n\\n`` stop separates blocks).
Three registered variants per language:
- ``MATHMinerva{LANG}_OLMES`` / ``MATHMinerva{LANG}_OLMES_NONL``: free-form, differ only in stop sequences.
- ``MATHMinerva{LANG}_BPB_OLMES``: bits-per-byte of the single gold solution (same prompt + few-shot).
German has 4947 samples; French has only 100.
"""
from dataclasses import dataclass
from typing import Any, Literal, final, get_args, override
from eval_framework.answer import AnswerPolicy, ExtractFromCompletion, PickFromCandidates
from eval_framework.benchmarks.utils.ellamind_languages import ellamind_language
from eval_framework.choices import ChoiceFields, ChoiceReader
from eval_framework.composed import ComposedBenchmark
from eval_framework.contract import Benchmark
from eval_framework.eval_kind import Choice, EvalKind, Generative
from eval_framework.fewshot import FewShot, FewshotExample, FunctionRenderer, SampleSplit
from eval_framework.metrics.base import BaseMetric
from eval_framework.metrics.completion.math_minerva_completion import (
MathMinervaCompletionDE,
MathMinervaCompletionFR,
MathMinervaCompletionRelaxedDE,
MathMinervaCompletionRelaxedFR,
)
from eval_framework.metrics.completion.minerva_math_utils import extract_answers, normalized_gold_from_solution
from eval_framework.subjects import ListOfSubjects
from eval_framework.tasks.dataset_loading import DatasetPolicy
from eval_framework.tasks.dataset_revisions import pinned_by_framework
from eval_framework.tasks.task_style import BPBStyle
HENDRYCKS_MATH_ELLAMIND_DATASET_PATH = "ellamind/hendrycks-math-multilingual"
_MAX_TOKENS = 1024
TaskVariant = Literal["OLMES", "OLMES_NONL", "BPB"]
LanguageCode = Literal["DE", "FR"]
_LANGUAGES: tuple[LanguageCode, ...] = get_args(LanguageCode)
# Variant -> task-id template. The ids predate the naming convention and are kept for continuity.
_TASK_ID_TEMPLATES: dict[TaskVariant, str] = {
"OLMES": "MATHMinerva{language}_OLMES",
"OLMES_NONL": "MATHMinerva{language}_OLMES_NONL",
"BPB": "MATHMinerva{language}_BPB_OLMES",
}
[docs]
@dataclass(frozen=True, kw_only=True)
class MinervaLocale:
"""One language's wording of the Minerva prompt, and the extraction and metrics that understand it."""
problem_prefix: str
"""Opens the problem block (``Problem:``); also the stop sequence of a free-form generation."""
solution_cue: str
"""Primes the solution turn (``Solution:``)."""
final_answer_line: str
"""Closes a shown solution; ``{answer}`` is the item's extracted answer. Its wording must be what the
language's ``cot_style`` extractor parses."""
cot_style: str
"""Key into ``minerva_math_utils.COT_EXTRACTORS``."""
metrics: list[type[BaseMetric]]
"""The strict and relaxed Minerva metrics for ``cot_style``."""
def __post_init__(self) -> None:
if "{answer}" not in self.final_answer_line:
raise ValueError(f"final_answer_line must contain '{{answer}}': {self.final_answer_line!r}")
_LOCALES: dict[LanguageCode, MinervaLocale] = {
"DE": MinervaLocale(
problem_prefix="Aufgabe:",
solution_cue="Lösung:",
final_answer_line="Finale Antwort: Die finale Antwort lautet ${answer}$. Ich hoffe, die Antwort ist korrekt.",
cot_style="minerva_de",
metrics=[MathMinervaCompletionDE, MathMinervaCompletionRelaxedDE],
),
"FR": MinervaLocale(
problem_prefix="Problème :",
solution_cue="Solution :",
final_answer_line="Réponse finale : La réponse finale est ${answer}$. J'espère que la réponse est correcte.",
cot_style="minerva_fr",
metrics=[MathMinervaCompletionFR, MathMinervaCompletionRelaxedFR],
),
}
@final
class _MinervaPrompt:
"""Assembles the Minerva prompt pieces in one language — shared by the generative and BPB variants."""
def __init__(self, locale: MinervaLocale) -> None:
self._locale = locale
def problem(self, item: dict[str, Any]) -> str:
return f"{self._locale.problem_prefix}\n{item['problem']}\n\n"
def solution(self, item: dict[str, Any]) -> str:
"""The gold solution closed by the final-answer line: the BPB target and the body of a demonstration."""
return f"{item['solution']}\n{self._locale.final_answer_line.format(answer=item['extracted_answer'])}"
def demo(self, item: dict[str, Any]) -> FewshotExample:
return FewshotExample(prompt=self.problem(item), answer=f"{self._locale.solution_cue} {self.solution(item)}")
def extract(self, completion_text: str) -> str:
candidates = extract_answers(completion_text, use_cot=True, cot_style=self._locale.cot_style, relaxed=True)
return candidates[0] if candidates else "[no_answer]"
def _minerva_gold(item: dict[str, Any]) -> str:
# "" reads the same as None to the minerva metric; keeps the gold a str (see math_reasoning._minerva_gold).
return normalized_gold_from_solution(item["solution"]) or ""
def _single_paragraph_solution(example: dict[str, Any]) -> bool:
# Keep only single-paragraph solutions: the "\n\n" stop must separate few-shot blocks, not fire inside one.
return "\n\n" not in example["solution"]
@final
class _MinervaBpbStyler(BPBStyle):
"""BPB scoring of the single gold solution, but under the Minerva prompt (problem prefix on its own line,
blank line before the cue) rather than the cloze ``question_prefix`` layout."""
def __init__(self, locale: MinervaLocale) -> None:
super().__init__(cue_text=locale.solution_cue)
self._problem_prefix = locale.problem_prefix
@override
def get_instruction_text(self, raw_question: str, choices: list[str]) -> str:
return f"{self._problem_prefix}\n{raw_question}\n\n"
@final
class _MinervaBpbReader(ChoiceReader):
"""The single scored 'choice' is the gold solution plus the language's final-answer line."""
def __init__(self, prompt: _MinervaPrompt) -> None:
self._prompt = prompt
@override
def read(self, item: dict[str, Any]) -> ChoiceFields:
return ChoiceFields(raw_question=item["problem"], choices=[self._prompt.solution(item)], correct_index=0)
[docs]
def hendrycks_math_ellamind(
task_variant: TaskVariant, language: LanguageCode, dataset: DatasetPolicy | None = None
) -> Benchmark:
"""Build ``MATHMinerva{language}_{variant}``, e.g. ``hendrycks_math_ellamind("OLMES", "DE")``."""
locale = _LOCALES[language]
prompt = _MinervaPrompt(locale)
config = ellamind_language(language).config
kind: EvalKind
answer: AnswerPolicy
if task_variant == "BPB":
# Same prompt and few-shot as the generative variants; only the eval turn changes to a scored gold solution.
kind = Choice(_MinervaBpbReader(prompt), _MinervaBpbStyler(locale))
answer = PickFromCandidates()
else:
# OLMES stops at the next problem or a blank line; OLMES_NONL drops the blank-line stop.
stop_sequences = [locale.problem_prefix] if task_variant == "OLMES_NONL" else [locale.problem_prefix, "\n\n"]
kind = Generative(
build_prompt=prompt.problem,
cue=locale.solution_cue,
ground_truth=_minerva_gold,
metrics=locale.metrics,
)
answer = ExtractFromCompletion(prompt.extract, stop_sequences, max_tokens=_MAX_TOKENS)
return ComposedBenchmark.compose(
id=_TASK_ID_TEMPLATES[task_variant].format(language=language),
kind=kind,
answer=answer,
sample_split="test",
fewshot=FewShot(SampleSplit(keep=_single_paragraph_solution), FunctionRenderer(prompt.demo)),
subjects=ListOfSubjects([config]),
dataset_policy=dataset if dataset is not None else pinned_by_framework(HENDRYCKS_MATH_ELLAMIND_DATASET_PATH),
)
HENDRYCKS_MATH_ELLAMIND_BENCHMARKS: list[Benchmark] = [
hendrycks_math_ellamind(task_variant, language) for language in _LANGUAGES for task_variant in get_args(TaskVariant)
]