Source code for eval_framework.benchmarks.social_iqa

"""Social IQa (English): https://huggingface.co/datasets/allenai/social_i_qa

Commonsense reasoning about social situations: a context and question with three answer options
(``answerA``/``answerB``/``answerC``) and a 1-indexed ``label`` marking the correct one. The registered
variant is OLMES-style: options are shown as space-prefixed lettered choices (" A. …") and the model is
scored over the letter labels.

``allenai/social_i_qa`` ships only a (no-longer-supported) loading script on its main branch, so its
frozen pin in ``frozen-hf-dataset-revisions.json`` points at a commit on Hugging Face's auto-generated
``refs/convert/parquet`` branch.
"""

from typing import Any, final, override

from eval_framework.choices import ChoiceFields, ChoiceReader
from eval_framework.composed import ComposedBenchmark
from eval_framework.contract import Benchmark
from eval_framework.tasks.dataset_loading import DatasetPolicy
from eval_framework.tasks.dataset_revisions import pinned_frozen
from eval_framework.tasks.task_style import MCStyle

SOCIAL_I_QA_DATASET_PATH = "allenai/social_i_qa"


[docs] @final class SocialIqaReader(ChoiceReader): """Reads a Social IQa item: the shown question is the context followed by the question, with the correct option at the 1-indexed ``label``."""
[docs] @override def read(self, item: dict[str, Any]) -> ChoiceFields: return ChoiceFields( raw_question=f"{item['context']} {item['question']}".strip(), choices=[item["answerA"], item["answerB"], item["answerC"]], correct_index=int(item["label"]) - 1, )
[docs] def social_iqa_mc_olmes(dataset: DatasetPolicy | None = None) -> Benchmark: styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True) dataset_policy = dataset if dataset is not None else pinned_frozen(SOCIAL_I_QA_DATASET_PATH) return ComposedBenchmark.choice( id="SocialIQAMC_OLMES", reader=SocialIqaReader(), styler=styler, sample_split="train", fewshot_split="train", dataset_policy=dataset_policy, )
SOCIAL_IQA_BENCHMARKS: list[Benchmark] = [social_iqa_mc_olmes()]