Source code for eval_framework.tasks.task_names

from enum import Enum

from eval_framework.benchmarks.arc import ARC_BENCHMARKS
from eval_framework.benchmarks.arc_de import ARC_DE_BENCHMARK
from eval_framework.benchmarks.arc_ellamind import ARC_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.bigcodebench import BIGCODEBENCH_BENCHMARKS
from eval_framework.benchmarks.copa import COPA_BENCHMARKS
from eval_framework.benchmarks.csqa import CSQA_BENCHMARKS
from eval_framework.benchmarks.csqa_ellamind import CSQA_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.drop import DROP_BENCHMARKS
from eval_framework.benchmarks.global_mmlu import GLOBAL_MMLU_BENCHMARKS
from eval_framework.benchmarks.goldenswag import GOLDENSWAG_BENCHMARKS
from eval_framework.benchmarks.gpqa import GPQA_BENCHMARKS
from eval_framework.benchmarks.gpqa_ellamind import GPQA_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.gsm8k import GSM8K_BENCHMARKS
from eval_framework.benchmarks.gsm8k_ellamind import GSM8K_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.hellaswag import HELLASWAG_BENCHMARKS
from eval_framework.benchmarks.hellaswag_ellamind import HELLASWAG_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.hendrycks_math_ellamind import HENDRYCKS_MATH_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.hle_ellamind import HLE_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.humaneval import HUMANEVAL_BENCHMARKS
from eval_framework.benchmarks.humaneval_ellamind import HUMANEVAL_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.humaneval_plus import HUMANEVAL_PLUS_BENCHMARKS
from eval_framework.benchmarks.ifeval import IFEVAL_BENCHMARKS
from eval_framework.benchmarks.math_reasoning import MATH_REASONING_BENCHMARKS
from eval_framework.benchmarks.mbpp import MBPP_BENCHMARKS
from eval_framework.benchmarks.mbpp_ellamind import MBPP_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.mmlu import MMLU_BENCHMARKS
from eval_framework.benchmarks.mmlu_pro import MMLU_PRO_BENCHMARKS
from eval_framework.benchmarks.multipl_e import MULTIPL_E_BENCHMARKS
from eval_framework.benchmarks.naturalqs_open import NATURALQS_OPEN_BENCHMARKS
from eval_framework.benchmarks.piqa import PIQA_BENCHMARKS
from eval_framework.benchmarks.piqa_ellamind import PIQA_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.sciq import SCIQ_BENCHMARKS
from eval_framework.benchmarks.simpleqa_ellamind import SIMPLEQA_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.siqa_ellamind import SIQA_ELLAMIND_BENCHMARKS
from eval_framework.benchmarks.social_iqa import SOCIAL_IQA_BENCHMARKS
from eval_framework.benchmarks.squad import SQUAD_BENCHMARKS
from eval_framework.benchmarks.winogrande import WINOGRANDE_BENCHMARKS
from eval_framework.benchmarks.winogrande_ellamind import WINOGRANDE_ELLAMIND_BENCHMARKS
from eval_framework.tasks.base import BaseTask
from eval_framework.tasks.registry import Registry
from eval_framework.tasks.registry import registry as global_registry




class TaskNameEnum(Enum):
    @property
    def value(self) -> type[BaseTask]:
        return super().value





def register_all_tasks(registry: Registry | None = None) -> None:
    """Register all the benchmark tasks with the eval framework

    Uses global registry by default.
    """
    registry = registry if registry is not None else global_registry()

    register_math_reasoning_tasks(registry=registry)
    register_arc_tasks(registry=registry)
    register_arc_de_tasks(registry=registry)
    register_bigcodebench_tasks(registry=registry)
    register_copa_tasks(registry=registry)
    register_goldenswag_tasks(registry=registry)
    register_gpqa_tasks(registry=registry)
    register_gsm8k_tasks(registry=registry)
    register_hellaswag_tasks(registry=registry)
    register_humaneval_tasks(registry=registry)
    register_humaneval_plus_tasks(registry=registry)
    register_ifeval_tasks(registry=registry)
    register_multipl_e_tasks(registry=registry)
    register_mbpp_tasks(registry=registry)
    register_mmlu_tasks(registry=registry)
    register_mmlu_pro_tasks(registry=registry)
    register_global_mmlu_tasks(registry=registry)
    register_piqa_tasks(registry=registry)
    register_sciq_tasks(registry=registry)
    register_squad_tasks(registry=registry)
    register_winogrande_tasks(registry=registry)
    register_csqa_tasks(registry=registry)
    register_drop_tasks(registry=registry)
    register_naturalqs_open_tasks(registry=registry)
    register_social_iqa_tasks(registry=registry)
    register_arc_ellamind_tasks(registry=registry)
    register_csqa_ellamind_tasks(registry=registry)
    register_gpqa_ellamind_tasks(registry=registry)
    register_gsm8k_ellamind_tasks(registry=registry)
    register_hellaswag_ellamind_tasks(registry=registry)
    register_hendrycks_math_ellamind_tasks(registry=registry)
    register_hle_ellamind_tasks(registry=registry)
    register_humaneval_ellamind_tasks(registry=registry)
    register_mbpp_ellamind_tasks(registry=registry)
    register_piqa_ellamind_tasks(registry=registry)
    register_simpleqa_ellamind_tasks(registry=registry)
    register_siqa_ellamind_tasks(registry=registry)
    register_winogrande_ellamind_tasks(registry=registry)





def register_arc_tasks(registry: Registry) -> None:
    """Register arc benchmark tasks."""
    for benchmark in ARC_BENCHMARKS:
        registry.add(benchmark)





def register_hellaswag_tasks(registry: Registry) -> None:
    """Register hellaswag benchmark tasks."""
    for benchmark in HELLASWAG_BENCHMARKS:
        registry.add(benchmark)





def register_piqa_tasks(registry: Registry) -> None:
    """Register piqa benchmark tasks."""
    for benchmark in PIQA_BENCHMARKS:
        registry.add(benchmark)





def register_gpqa_tasks(registry: Registry) -> None:
    """Register gpqa benchmark tasks."""
    for benchmark in GPQA_BENCHMARKS:
        registry.add(benchmark)





def register_gsm8k_tasks(registry: Registry) -> None:
    """Register gsm8k benchmark tasks."""
    for benchmark in GSM8K_BENCHMARKS:
        registry.add(benchmark)





def register_math_reasoning_tasks(registry: Registry) -> None:
    """Register math_reasoning benchmark tasks (all composed: MATH500(_V2), AIME, GSM8KReasoning, Minerva-OLMES)."""
    for benchmark in MATH_REASONING_BENCHMARKS:
        registry.add(benchmark)





def register_mmlu_tasks(registry: Registry) -> None:
    """Register mmlu benchmark tasks."""
    for benchmark in MMLU_BENCHMARKS:  # composed: MMLU, MMLU_OLMES, FullTextMMLU, MMLU_IDK, MMLU_COT
        registry.add(benchmark)





def register_humaneval_tasks(registry: Registry) -> None:
    """Register humaneval benchmark tasks."""
    for benchmark in HUMANEVAL_BENCHMARKS:
        registry.add(benchmark)





def register_humaneval_plus_tasks(registry: Registry) -> None:
    """Register humaneval_plus benchmark tasks."""
    for benchmark in HUMANEVAL_PLUS_BENCHMARKS:
        registry.add(benchmark)





def register_mbpp_tasks(registry: Registry) -> None:
    """Register mbpp benchmark tasks."""
    for benchmark in MBPP_BENCHMARKS:
        registry.add(benchmark)





def register_bigcodebench_tasks(registry: Registry) -> None:
    """Register bigcodebench benchmark tasks."""
    for benchmark in BIGCODEBENCH_BENCHMARKS:
        registry.add(benchmark)





def register_arc_de_tasks(registry: Registry) -> None:
    """Register arc_de benchmark tasks."""
    registry.add(ARC_DE_BENCHMARK)





def register_copa_tasks(registry: Registry) -> None:
    """Register copa benchmark tasks."""
    for benchmark in COPA_BENCHMARKS:
        registry.add(benchmark)





def register_goldenswag_tasks(registry: Registry) -> None:
    """Register goldenswag benchmark tasks."""
    for benchmark in GOLDENSWAG_BENCHMARKS:
        registry.add(benchmark)





def register_ifeval_tasks(registry: Registry) -> None:
    """Register ifeval benchmark tasks (composed: IFEval, IFEvalDe)."""
    for benchmark in IFEVAL_BENCHMARKS:
        registry.add(benchmark)





def register_multipl_e_tasks(registry: Registry) -> None:
    """Register multipl_e benchmark tasks."""
    for benchmark in MULTIPL_E_BENCHMARKS:
        registry.add(benchmark)





def register_mmlu_pro_tasks(registry: Registry) -> None:
    """Register mmlu_pro benchmark tasks."""
    for benchmark in MMLU_PRO_BENCHMARKS:
        registry.add(benchmark)





def register_global_mmlu_tasks(registry: Registry) -> None:
    """Register global_mmlu benchmark tasks."""
    for benchmark in GLOBAL_MMLU_BENCHMARKS:
        registry.add(benchmark)





def register_sciq_tasks(registry: Registry) -> None:
    """Register sciq benchmark tasks."""
    for benchmark in SCIQ_BENCHMARKS:
        registry.add(benchmark)





def register_squad_tasks(registry: Registry) -> None:
    """Register squad benchmark tasks (composed: SQuAD_OLMES, SQuAD2_MA, SQuAD2_MA_NO_SYSPROMPT)."""
    for benchmark in SQUAD_BENCHMARKS:
        registry.add(benchmark)





def register_winogrande_tasks(registry: Registry) -> None:
    """Register winogrande benchmark tasks."""
    for benchmark in WINOGRANDE_BENCHMARKS:
        registry.add(benchmark)





def register_csqa_tasks(registry: Registry) -> None:
    """Register csqa benchmark tasks."""
    for benchmark in CSQA_BENCHMARKS:
        registry.add(benchmark)





def register_drop_tasks(registry: Registry) -> None:
    """Register drop benchmark tasks (composed: DropCompletion_OLMES, DropMC_OLMES)."""
    for benchmark in DROP_BENCHMARKS:
        registry.add(benchmark)





def register_naturalqs_open_tasks(registry: Registry) -> None:
    """Register naturalqs_open benchmark tasks (composed: NaturalQsOpen, NaturalQsOpenMC_OLMES)."""
    for benchmark in NATURALQS_OPEN_BENCHMARKS:
        registry.add(benchmark)





def register_social_iqa_tasks(registry: Registry) -> None:
    """Register social_iqa benchmark tasks."""
    for benchmark in SOCIAL_IQA_BENCHMARKS:
        registry.add(benchmark)





def register_arc_ellamind_tasks(registry: Registry) -> None:
    """Register arc_ellamind benchmark tasks."""
    for benchmark in ARC_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_csqa_ellamind_tasks(registry: Registry) -> None:
    """Register csqa_ellamind benchmark tasks."""
    for benchmark in CSQA_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_gpqa_ellamind_tasks(registry: Registry) -> None:
    """Register gpqa_ellamind benchmark tasks."""
    for benchmark in GPQA_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_gsm8k_ellamind_tasks(registry: Registry) -> None:
    """Register gsm8k_ellamind benchmark tasks."""
    for benchmark in GSM8K_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_hellaswag_ellamind_tasks(registry: Registry) -> None:
    """Register hellaswag_ellamind benchmark tasks."""
    for benchmark in HELLASWAG_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_hendrycks_math_ellamind_tasks(registry: Registry) -> None:
    """Register hendrycks_math_ellamind benchmark tasks (composed)."""
    for benchmark in HENDRYCKS_MATH_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_hle_ellamind_tasks(registry: Registry) -> None:
    """Register hle_ellamind benchmark tasks."""
    for benchmark in HLE_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_humaneval_ellamind_tasks(registry: Registry) -> None:
    """Register humaneval_ellamind benchmark tasks."""
    for benchmark in HUMANEVAL_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_mbpp_ellamind_tasks(registry: Registry) -> None:
    """Register mbpp_ellamind benchmark tasks."""
    for benchmark in MBPP_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_piqa_ellamind_tasks(registry: Registry) -> None:
    """Register piqa_ellamind benchmark tasks."""
    for benchmark in PIQA_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_simpleqa_ellamind_tasks(registry: Registry) -> None:
    """Register simpleqa_ellamind benchmark tasks."""
    for benchmark in SIMPLEQA_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_siqa_ellamind_tasks(registry: Registry) -> None:
    """Register siqa_ellamind benchmark tasks."""
    for benchmark in SIQA_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)





def register_winogrande_ellamind_tasks(registry: Registry) -> None:
    """Register winogrande_ellamind benchmark tasks."""
    for benchmark in WINOGRANDE_ELLAMIND_BENCHMARKS:
        registry.add(benchmark)