Index A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z A AAA (eval_framework.tasks.base.Language attribute) AAB (eval_framework.tasks.base.Language attribute) AAC (eval_framework.tasks.base.Language attribute) AAD (eval_framework.tasks.base.Language attribute) AAE (eval_framework.tasks.base.Language attribute) AAF (eval_framework.tasks.base.Language attribute) AAG (eval_framework.tasks.base.Language attribute) AAH (eval_framework.tasks.base.Language attribute) AAI (eval_framework.tasks.base.Language attribute) AAK (eval_framework.tasks.base.Language attribute) AAL (eval_framework.tasks.base.Language attribute) AAM (eval_framework.tasks.base.Language attribute) AAN (eval_framework.tasks.base.Language attribute) AAO (eval_framework.tasks.base.Language attribute) AAP (eval_framework.tasks.base.Language attribute) AAQ (eval_framework.tasks.base.Language attribute) AAR (eval_framework.tasks.base.Language attribute) AAS (eval_framework.tasks.base.Language attribute) AAT (eval_framework.tasks.base.Language attribute) AAU (eval_framework.tasks.base.Language attribute) AAW (eval_framework.tasks.base.Language attribute) AAX (eval_framework.tasks.base.Language attribute) AAY (eval_framework.tasks.base.Language attribute) AAZ (eval_framework.tasks.base.Language attribute) ABA (eval_framework.tasks.base.Language attribute) ABB (eval_framework.tasks.base.Language attribute) ABC (eval_framework.tasks.base.Language attribute) ABD (eval_framework.tasks.base.Language attribute) ABE (eval_framework.tasks.base.Language attribute) ABF (eval_framework.tasks.base.Language attribute) ABG (eval_framework.tasks.base.Language attribute) ABH (eval_framework.tasks.base.Language attribute) ABI (eval_framework.tasks.base.Language attribute) ABJ (eval_framework.tasks.base.Language attribute) ABK (eval_framework.tasks.base.Language attribute) ABL (eval_framework.tasks.base.Language attribute) ABM (eval_framework.tasks.base.Language attribute) ABN (eval_framework.tasks.base.Language attribute) ABO (eval_framework.tasks.base.Language attribute) ABP (eval_framework.tasks.base.Language attribute) ABQ (eval_framework.tasks.base.Language attribute) ABR (eval_framework.tasks.base.Language attribute) ABS (eval_framework.tasks.base.Language attribute) ABT (eval_framework.tasks.base.Language attribute) ABU (eval_framework.tasks.base.Language attribute) ABV (eval_framework.tasks.base.Language attribute) ABW (eval_framework.tasks.base.Language attribute) ABX (eval_framework.tasks.base.Language attribute) ABY (eval_framework.tasks.base.Language attribute) ABZ (eval_framework.tasks.base.Language attribute) ACA (eval_framework.tasks.base.Language attribute) ACB (eval_framework.tasks.base.Language attribute) ACC (eval_framework.tasks.base.Language attribute) AccuracyCompletion (class in eval_framework.metrics.completion.accuracy_completion) AccuracyLoglikelihood (class in eval_framework.metrics.loglikelihood.accuracy_loglikelihood) AccuracyNormLoglikelihood (class in eval_framework.metrics.loglikelihood.accuracy_loglikelihood) ACD (eval_framework.tasks.base.Language attribute) ACE (eval_framework.tasks.base.Language attribute) ACF (eval_framework.tasks.base.Language attribute) ACH (eval_framework.tasks.base.Language attribute) ACI (eval_framework.tasks.base.Language attribute) ACK (eval_framework.tasks.base.Language attribute) ACL (eval_framework.tasks.base.Language attribute) ACM (eval_framework.tasks.base.Language attribute) ACN (eval_framework.tasks.base.Language attribute) ACP (eval_framework.tasks.base.Language attribute) ACQ (eval_framework.tasks.base.Language attribute) ACR (eval_framework.tasks.base.Language attribute) ACS (eval_framework.tasks.base.Language attribute) ACT (eval_framework.tasks.base.Language attribute) ACU (eval_framework.tasks.base.Language attribute) ACV (eval_framework.tasks.base.Language attribute) ACW (eval_framework.tasks.base.Language attribute) ACX (eval_framework.tasks.base.Language attribute) ACY (eval_framework.tasks.base.Language attribute) ACZ (eval_framework.tasks.base.Language attribute) ADA (eval_framework.tasks.base.Language attribute) ADB (eval_framework.tasks.base.Language attribute) ADD (eval_framework.tasks.base.Language attribute) add() (eval_framework.tasks.registry.Registry method) additional_kwargs (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) ADE (eval_framework.tasks.base.Language attribute) ADF (eval_framework.tasks.base.Language attribute) ADG (eval_framework.tasks.base.Language attribute) ADH (eval_framework.tasks.base.Language attribute) ADI (eval_framework.tasks.base.Language attribute) ADJ (eval_framework.tasks.base.Language attribute) ADL (eval_framework.tasks.base.Language attribute) ADN (eval_framework.tasks.base.Language attribute) ADO (eval_framework.tasks.base.Language attribute) ADP (eval_framework.tasks.base.Language attribute) ADQ (eval_framework.tasks.base.Language attribute) ADR (eval_framework.tasks.base.Language attribute) ADS (eval_framework.tasks.base.Language attribute) ADT (eval_framework.tasks.base.Language attribute) ADU (eval_framework.tasks.base.Language attribute) ADW (eval_framework.tasks.base.Language attribute) ADX (eval_framework.tasks.base.Language attribute) ADY (eval_framework.tasks.base.Language attribute) ADZ (eval_framework.tasks.base.Language attribute) AEA (eval_framework.tasks.base.Language attribute) AEB (eval_framework.tasks.base.Language attribute) AEC (eval_framework.tasks.base.Language attribute) AED (eval_framework.tasks.base.Language attribute) AEE (eval_framework.tasks.base.Language attribute) AEK (eval_framework.tasks.base.Language attribute) AEL (eval_framework.tasks.base.Language attribute) AEM (eval_framework.tasks.base.Language attribute) AEN (eval_framework.tasks.base.Language attribute) AEQ (eval_framework.tasks.base.Language attribute) AER (eval_framework.tasks.base.Language attribute) AES (eval_framework.tasks.base.Language attribute) AEU (eval_framework.tasks.base.Language attribute) AEW (eval_framework.tasks.base.Language attribute) AEX (eval_framework.tasks.base.Language attribute) AEY (eval_framework.tasks.base.Language attribute) AEZ (eval_framework.tasks.base.Language attribute) AFB (eval_framework.tasks.base.Language attribute) AFD (eval_framework.tasks.base.Language attribute) AFE (eval_framework.tasks.base.Language attribute) AFG (eval_framework.tasks.base.Language attribute) AFH (eval_framework.tasks.base.Language attribute) AFI (eval_framework.tasks.base.Language attribute) AFK (eval_framework.tasks.base.Language attribute) AFN (eval_framework.tasks.base.Language attribute) AFO (eval_framework.tasks.base.Language attribute) AFP (eval_framework.tasks.base.Language attribute) AFR (eval_framework.tasks.base.Language attribute) AFS (eval_framework.tasks.base.Language attribute) AFT (eval_framework.tasks.base.Language attribute) AFU (eval_framework.tasks.base.Language attribute) AFZ (eval_framework.tasks.base.Language attribute) AGA (eval_framework.tasks.base.Language attribute) AGB (eval_framework.tasks.base.Language attribute) AGC (eval_framework.tasks.base.Language attribute) AGD (eval_framework.tasks.base.Language attribute) AGE (eval_framework.tasks.base.Language attribute) AGF (eval_framework.tasks.base.Language attribute) AGG (eval_framework.tasks.base.Language attribute) AGH (eval_framework.tasks.base.Language attribute) AGI (eval_framework.tasks.base.Language attribute) AGJ (eval_framework.tasks.base.Language attribute) AGK (eval_framework.tasks.base.Language attribute) AGL (eval_framework.tasks.base.Language attribute) AGM (eval_framework.tasks.base.Language attribute) AGN (eval_framework.tasks.base.Language attribute) AGO (eval_framework.tasks.base.Language attribute) AGP (eval_framework.tasks.base.Language attribute) AGQ (eval_framework.tasks.base.Language attribute) AGR (eval_framework.tasks.base.Language attribute) AGS (eval_framework.tasks.base.Language attribute) AGT (eval_framework.tasks.base.Language attribute) AGU (eval_framework.tasks.base.Language attribute) AGV (eval_framework.tasks.base.Language attribute) AGW (eval_framework.tasks.base.Language attribute) AGX (eval_framework.tasks.base.Language attribute) AGY (eval_framework.tasks.base.Language attribute) AGZ (eval_framework.tasks.base.Language attribute) AHA (eval_framework.tasks.base.Language attribute) AHB (eval_framework.tasks.base.Language attribute) AHE (eval_framework.tasks.base.Language attribute) AHG (eval_framework.tasks.base.Language attribute) AHH (eval_framework.tasks.base.Language attribute) AHI (eval_framework.tasks.base.Language attribute) AHK (eval_framework.tasks.base.Language attribute) AHL (eval_framework.tasks.base.Language attribute) AHM (eval_framework.tasks.base.Language attribute) AHN (eval_framework.tasks.base.Language attribute) AHO (eval_framework.tasks.base.Language attribute) AHP (eval_framework.tasks.base.Language attribute) AHR (eval_framework.tasks.base.Language attribute) AHS (eval_framework.tasks.base.Language attribute) AHT (eval_framework.tasks.base.Language attribute) AIA (eval_framework.tasks.base.Language attribute) AIB (eval_framework.tasks.base.Language attribute) AIC (eval_framework.tasks.base.Language attribute) AID (eval_framework.tasks.base.Language attribute) AidanBench (class in eval_framework.tasks.benchmarks.aidanbench) AidanBenchMetric (class in eval_framework.metrics.completion.aidanbench) AidanBenchOriginal (class in eval_framework.tasks.benchmarks.aidanbench) AIE (eval_framework.tasks.base.Language attribute) AIF (eval_framework.tasks.base.Language attribute) AIG (eval_framework.tasks.base.Language attribute) AIH (eval_framework.tasks.base.Language attribute) AII (eval_framework.tasks.base.Language attribute) AIJ (eval_framework.tasks.base.Language attribute) AIK (eval_framework.tasks.base.Language attribute) AIL (eval_framework.tasks.base.Language attribute) AIM (eval_framework.tasks.base.Language attribute) AIME2024 (class in eval_framework.tasks.benchmarks.math_reasoning) AIME2025 (class in eval_framework.tasks.benchmarks.math_reasoning) AIN (eval_framework.tasks.base.Language attribute) AIO (eval_framework.tasks.base.Language attribute) AIP (eval_framework.tasks.base.Language attribute) AIQ (eval_framework.tasks.base.Language attribute) AIR (eval_framework.tasks.base.Language attribute) AIS (eval_framework.tasks.base.Language attribute) AIT (eval_framework.tasks.base.Language attribute) AIW (eval_framework.tasks.base.Language attribute) AIX (eval_framework.tasks.base.Language attribute) AIY (eval_framework.tasks.base.Language attribute) AIZ (eval_framework.tasks.base.Language attribute) AJA (eval_framework.tasks.base.Language attribute) AJG (eval_framework.tasks.base.Language attribute) AJI (eval_framework.tasks.base.Language attribute) AJN (eval_framework.tasks.base.Language attribute) AJP (eval_framework.tasks.base.Language attribute) AJS (eval_framework.tasks.base.Language attribute) AJT (eval_framework.tasks.base.Language attribute) AJU (eval_framework.tasks.base.Language attribute) AJW (eval_framework.tasks.base.Language attribute) AJZ (eval_framework.tasks.base.Language attribute) AKA (eval_framework.tasks.base.Language attribute) AKB (eval_framework.tasks.base.Language attribute) AKC (eval_framework.tasks.base.Language attribute) AKD (eval_framework.tasks.base.Language attribute) AKE (eval_framework.tasks.base.Language attribute) AKF (eval_framework.tasks.base.Language attribute) AKG (eval_framework.tasks.base.Language attribute) AKH (eval_framework.tasks.base.Language attribute) AKI (eval_framework.tasks.base.Language attribute) AKJ (eval_framework.tasks.base.Language attribute) AKK (eval_framework.tasks.base.Language attribute) AKL (eval_framework.tasks.base.Language attribute) AKM (eval_framework.tasks.base.Language attribute) AKN (eval_framework.tasks.base.Language attribute) AKO (eval_framework.tasks.base.Language attribute) AKP (eval_framework.tasks.base.Language attribute) AKQ (eval_framework.tasks.base.Language attribute) AKR (eval_framework.tasks.base.Language attribute) AKS (eval_framework.tasks.base.Language attribute) AKT (eval_framework.tasks.base.Language attribute) AKU (eval_framework.tasks.base.Language attribute) AKV (eval_framework.tasks.base.Language attribute) AKW (eval_framework.tasks.base.Language attribute) AKX (eval_framework.tasks.base.Language attribute) AKY (eval_framework.tasks.base.Language attribute) AKZ (eval_framework.tasks.base.Language attribute) ALA (eval_framework.tasks.base.Language attribute) ALC (eval_framework.tasks.base.Language attribute) ALD (eval_framework.tasks.base.Language attribute) ALE (eval_framework.tasks.base.Language attribute) AlephAlphaAPIModel (class in eval_framework.llm.aleph_alpha) ALF (eval_framework.tasks.base.Language attribute) ALH (eval_framework.tasks.base.Language attribute) ALI (eval_framework.tasks.base.Language attribute) ALJ (eval_framework.tasks.base.Language attribute) ALK (eval_framework.tasks.base.Language attribute) ALL (eval_framework.tasks.base.Language attribute) ALM (eval_framework.tasks.base.Language attribute) ALN (eval_framework.tasks.base.Language attribute) ALO (eval_framework.tasks.base.Language attribute) ALP (eval_framework.tasks.base.Language attribute) ALQ (eval_framework.tasks.base.Language attribute) ALR (eval_framework.tasks.base.Language attribute) ALS (eval_framework.tasks.base.Language attribute) ALT (eval_framework.tasks.base.Language attribute) ALU (eval_framework.tasks.base.Language attribute) ALW (eval_framework.tasks.base.Language attribute) ALX (eval_framework.tasks.base.Language attribute) ALY (eval_framework.tasks.base.Language attribute) ALZ (eval_framework.tasks.base.Language attribute) AMA (eval_framework.tasks.base.Language attribute) AMB (eval_framework.tasks.base.Language attribute) AMC (eval_framework.tasks.base.Language attribute) AMD (eval_framework.tasks.base.Language attribute) AME (eval_framework.tasks.base.Language attribute) AMF (eval_framework.tasks.base.Language attribute) AMG (eval_framework.tasks.base.Language attribute) AMH (eval_framework.tasks.base.Language attribute) AMI (eval_framework.tasks.base.Language attribute) AMJ (eval_framework.tasks.base.Language attribute) AMK (eval_framework.tasks.base.Language attribute) AML (eval_framework.tasks.base.Language attribute) AMM (eval_framework.tasks.base.Language attribute) AMN (eval_framework.tasks.base.Language attribute) AMO (eval_framework.tasks.base.Language attribute) AMP (eval_framework.tasks.base.Language attribute) AMQ (eval_framework.tasks.base.Language attribute) AMR (eval_framework.tasks.base.Language attribute) AMS (eval_framework.tasks.base.Language attribute) AMT (eval_framework.tasks.base.Language attribute) AMU (eval_framework.tasks.base.Language attribute) AMV (eval_framework.tasks.base.Language attribute) AMW (eval_framework.tasks.base.Language attribute) AMX (eval_framework.tasks.base.Language attribute) AMY (eval_framework.tasks.base.Language attribute) AMZ (eval_framework.tasks.base.Language attribute) ANA (eval_framework.tasks.base.Language attribute) ANB (eval_framework.tasks.base.Language attribute) ANC (eval_framework.tasks.base.Language attribute) AND (eval_framework.tasks.base.Language attribute) ANE (eval_framework.tasks.base.Language attribute) ANF (eval_framework.tasks.base.Language attribute) ANG (eval_framework.tasks.base.Language attribute) ANH (eval_framework.tasks.base.Language attribute) ANI (eval_framework.tasks.base.Language attribute) ANJ (eval_framework.tasks.base.Language attribute) ANK (eval_framework.tasks.base.Language attribute) ANL (eval_framework.tasks.base.Language attribute) ANM (eval_framework.tasks.base.Language attribute) ANN (eval_framework.tasks.base.Language attribute) ANO (eval_framework.tasks.base.Language attribute) ANP (eval_framework.tasks.base.Language attribute) ANQ (eval_framework.tasks.base.Language attribute) ANR (eval_framework.tasks.base.Language attribute) ANS (eval_framework.tasks.base.Language attribute) ANS_RE (eval_framework.tasks.benchmarks.gpqa.GPQA_COT attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) answer (eval_framework.metrics.llm.llm_judge_mtbench_pair.MTBenchJudgePairMetricContext attribute) ANSWER_PATTERN (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATHReasoning attribute) ANT (eval_framework.tasks.base.Language attribute) ANU (eval_framework.tasks.base.Language attribute) ANV (eval_framework.tasks.base.Language attribute) ANW (eval_framework.tasks.base.Language attribute) ANX (eval_framework.tasks.base.Language attribute) ANY (eval_framework.tasks.base.Language attribute) ANZ (eval_framework.tasks.base.Language attribute) AOA (eval_framework.tasks.base.Language attribute) AOB (eval_framework.tasks.base.Language attribute) AOC (eval_framework.tasks.base.Language attribute) AOD (eval_framework.tasks.base.Language attribute) AOE (eval_framework.tasks.base.Language attribute) AOF (eval_framework.tasks.base.Language attribute) AOG (eval_framework.tasks.base.Language attribute) AOH (eval_framework.tasks.base.Language attribute) AOI (eval_framework.tasks.base.Language attribute) AOJ (eval_framework.tasks.base.Language attribute) AOK (eval_framework.tasks.base.Language attribute) AOL (eval_framework.tasks.base.Language attribute) AOM (eval_framework.tasks.base.Language attribute) AON (eval_framework.tasks.base.Language attribute) AOR (eval_framework.tasks.base.Language attribute) AOS (eval_framework.tasks.base.Language attribute) AOT (eval_framework.tasks.base.Language attribute) AOU (eval_framework.tasks.base.Language attribute) AOX (eval_framework.tasks.base.Language attribute) AOZ (eval_framework.tasks.base.Language attribute) APB (eval_framework.tasks.base.Language attribute) APC (eval_framework.tasks.base.Language attribute) APD (eval_framework.tasks.base.Language attribute) APE (eval_framework.tasks.base.Language attribute) APF (eval_framework.tasks.base.Language attribute) APG (eval_framework.tasks.base.Language attribute) APH (eval_framework.tasks.base.Language attribute) API (eval_framework.tasks.base.Language attribute) APJ (eval_framework.tasks.base.Language attribute) APK (eval_framework.tasks.base.Language attribute) APL (eval_framework.tasks.base.Language attribute) APM (eval_framework.tasks.base.Language attribute) APN (eval_framework.tasks.base.Language attribute) APO (eval_framework.tasks.base.Language attribute) APP (eval_framework.tasks.base.Language attribute) APQ (eval_framework.tasks.base.Language attribute) APR (eval_framework.tasks.base.Language attribute) APS (eval_framework.tasks.base.Language attribute) APT (eval_framework.tasks.base.Language attribute) APU (eval_framework.tasks.base.Language attribute) APV (eval_framework.tasks.base.Language attribute) APW (eval_framework.tasks.base.Language attribute) APX (eval_framework.tasks.base.Language attribute) APY (eval_framework.tasks.base.Language attribute) APZ (eval_framework.tasks.base.Language attribute) AQC (eval_framework.tasks.base.Language attribute) AQD (eval_framework.tasks.base.Language attribute) AQG (eval_framework.tasks.base.Language attribute) AQK (eval_framework.tasks.base.Language attribute) AQM (eval_framework.tasks.base.Language attribute) AQN (eval_framework.tasks.base.Language attribute) AQP (eval_framework.tasks.base.Language attribute) AQR (eval_framework.tasks.base.Language attribute) AQT (eval_framework.tasks.base.Language attribute) AQZ (eval_framework.tasks.base.Language attribute) ARA (eval_framework.tasks.base.Language attribute) ARB (eval_framework.tasks.base.Language attribute) ARC (class in eval_framework.tasks.benchmarks.arc) (eval_framework.tasks.base.Language attribute) ARC_DE (class in eval_framework.tasks.benchmarks.arc_de) ARC_EU20_DE (class in eval_framework.tasks.benchmarks.opengptx_eu20) ARC_EU20_FR (class in eval_framework.tasks.benchmarks.opengptx_eu20) ARC_FI (class in eval_framework.tasks.benchmarks.arc_fi) ARC_IDK (class in eval_framework.tasks.benchmarks.arc) ARD (eval_framework.tasks.base.Language attribute) ARE (eval_framework.tasks.base.Language attribute) ARF (eval_framework.tasks.base.Language attribute) ARG (eval_framework.tasks.base.Language attribute) ARH (eval_framework.tasks.base.Language attribute) ARI (eval_framework.tasks.base.Language attribute) ARJ (eval_framework.tasks.base.Language attribute) ARK (eval_framework.tasks.base.Language attribute) ARL (eval_framework.tasks.base.Language attribute) ARN (eval_framework.tasks.base.Language attribute) ARO (eval_framework.tasks.base.Language attribute) ARP (eval_framework.tasks.base.Language attribute) ARQ (eval_framework.tasks.base.Language attribute) ARR (eval_framework.tasks.base.Language attribute) ARS (eval_framework.tasks.base.Language attribute) artifact_upload_function() (in module eval_framework.result_processors.wandb_uploader) ARU (eval_framework.tasks.base.Language attribute) ARV (eval_framework.tasks.base.Language attribute) ARW (eval_framework.tasks.base.Language attribute) ARX (eval_framework.tasks.base.Language attribute) ARY (eval_framework.tasks.base.Language attribute) ARZ (eval_framework.tasks.base.Language attribute) as_dict() (eval_framework.base_config.BaseConfig method) ASA (eval_framework.tasks.base.Language attribute) ASB (eval_framework.tasks.base.Language attribute) ASC (eval_framework.tasks.base.Language attribute) ASD (eval_framework.tasks.base.Language attribute) ASE (eval_framework.tasks.base.Language attribute) ASF (eval_framework.tasks.base.Language attribute) ASG (eval_framework.tasks.base.Language attribute) ASH (eval_framework.tasks.base.Language attribute) ASI (eval_framework.tasks.base.Language attribute) ASJ (eval_framework.tasks.base.Language attribute) ASK (eval_framework.tasks.base.Language attribute) ASL (eval_framework.tasks.base.Language attribute) ASM (eval_framework.tasks.base.Language attribute) ASN (eval_framework.tasks.base.Language attribute) ASO (eval_framework.tasks.base.Language attribute) ASP (eval_framework.tasks.base.Language attribute) ASQ (eval_framework.tasks.base.Language attribute) ASR (eval_framework.tasks.base.Language attribute) ASS (eval_framework.tasks.base.Language attribute) AST (eval_framework.tasks.base.Language attribute) ASU (eval_framework.tasks.base.Language attribute) ASV (eval_framework.tasks.base.Language attribute) ASW (eval_framework.tasks.base.Language attribute) ASX (eval_framework.tasks.base.Language attribute) ASY (eval_framework.tasks.base.Language attribute) ASZ (eval_framework.tasks.base.Language attribute) ATA (eval_framework.tasks.base.Language attribute) ATB (eval_framework.tasks.base.Language attribute) ATC (eval_framework.tasks.base.Language attribute) ATD (eval_framework.tasks.base.Language attribute) ATE (eval_framework.tasks.base.Language attribute) ATF (eval_framework.tasks.base.Language attribute) ATG (eval_framework.tasks.base.Language attribute) ATI (eval_framework.tasks.base.Language attribute) ATJ (eval_framework.tasks.base.Language attribute) ATK (eval_framework.tasks.base.Language attribute) ATL (eval_framework.tasks.base.Language attribute) ATM (eval_framework.tasks.base.Language attribute) ATN (eval_framework.tasks.base.Language attribute) ATO (eval_framework.tasks.base.Language attribute) ATP (eval_framework.tasks.base.Language attribute) ATQ (eval_framework.tasks.base.Language attribute) ATR (eval_framework.tasks.base.Language attribute) ATS (eval_framework.tasks.base.Language attribute) ATT (eval_framework.tasks.base.Language attribute) ATU (eval_framework.tasks.base.Language attribute) ATV (eval_framework.tasks.base.Language attribute) ATW (eval_framework.tasks.base.Language attribute) ATX (eval_framework.tasks.base.Language attribute) ATY (eval_framework.tasks.base.Language attribute) ATZ (eval_framework.tasks.base.Language attribute) AUA (eval_framework.tasks.base.Language attribute) AUB (eval_framework.tasks.base.Language attribute) AUC (eval_framework.tasks.base.Language attribute) AUD (eval_framework.tasks.base.Language attribute) AUE (eval_framework.tasks.base.Language attribute) AUG (eval_framework.tasks.base.Language attribute) AUH (eval_framework.tasks.base.Language attribute) AUI (eval_framework.tasks.base.Language attribute) AUJ (eval_framework.tasks.base.Language attribute) AUK (eval_framework.tasks.base.Language attribute) AUL (eval_framework.tasks.base.Language attribute) AUM (eval_framework.tasks.base.Language attribute) AUN (eval_framework.tasks.base.Language attribute) AUO (eval_framework.tasks.base.Language attribute) AUP (eval_framework.tasks.base.Language attribute) AUQ (eval_framework.tasks.base.Language attribute) AUR (eval_framework.tasks.base.Language attribute) AUT (eval_framework.tasks.base.Language attribute) AUU (eval_framework.tasks.base.Language attribute) AUV (eval_framework.tasks.base.Language attribute) AUW (eval_framework.tasks.base.Language attribute) AUX (eval_framework.tasks.base.Language attribute) AUY (eval_framework.tasks.base.Language attribute) AUZ (eval_framework.tasks.base.Language attribute) AVA (eval_framework.tasks.base.Language attribute) AVB (eval_framework.tasks.base.Language attribute) AVD (eval_framework.tasks.base.Language attribute) AVE (eval_framework.tasks.base.Language attribute) AVI (eval_framework.tasks.base.Language attribute) AVK (eval_framework.tasks.base.Language attribute) AVL (eval_framework.tasks.base.Language attribute) AVM (eval_framework.tasks.base.Language attribute) AVN (eval_framework.tasks.base.Language attribute) AVO (eval_framework.tasks.base.Language attribute) AVS (eval_framework.tasks.base.Language attribute) AVT (eval_framework.tasks.base.Language attribute) AVU (eval_framework.tasks.base.Language attribute) AVV (eval_framework.tasks.base.Language attribute) AWA (eval_framework.tasks.base.Language attribute) AWB (eval_framework.tasks.base.Language attribute) AWC (eval_framework.tasks.base.Language attribute) AWE (eval_framework.tasks.base.Language attribute) AWG (eval_framework.tasks.base.Language attribute) AWH (eval_framework.tasks.base.Language attribute) AWI (eval_framework.tasks.base.Language attribute) AWK (eval_framework.tasks.base.Language attribute) AWM (eval_framework.tasks.base.Language attribute) AWN (eval_framework.tasks.base.Language attribute) AWO (eval_framework.tasks.base.Language attribute) AWR (eval_framework.tasks.base.Language attribute) AWS (eval_framework.tasks.base.Language attribute) AWT (eval_framework.tasks.base.Language attribute) AWU (eval_framework.tasks.base.Language attribute) AWV (eval_framework.tasks.base.Language attribute) AWW (eval_framework.tasks.base.Language attribute) AWX (eval_framework.tasks.base.Language attribute) AWY (eval_framework.tasks.base.Language attribute) AXB (eval_framework.tasks.base.Language attribute) AXE (eval_framework.tasks.base.Language attribute) AXG (eval_framework.tasks.base.Language attribute) AXK (eval_framework.tasks.base.Language attribute) AXL (eval_framework.tasks.base.Language attribute) AXM (eval_framework.tasks.base.Language attribute) AXX (eval_framework.tasks.base.Language attribute) AYA (eval_framework.tasks.base.Language attribute) AYB (eval_framework.tasks.base.Language attribute) AYC (eval_framework.tasks.base.Language attribute) AYD (eval_framework.tasks.base.Language attribute) AYE (eval_framework.tasks.base.Language attribute) AYG (eval_framework.tasks.base.Language attribute) AYH (eval_framework.tasks.base.Language attribute) AYI (eval_framework.tasks.base.Language attribute) AYK (eval_framework.tasks.base.Language attribute) AYL (eval_framework.tasks.base.Language attribute) AYM (eval_framework.tasks.base.Language attribute) AYN (eval_framework.tasks.base.Language attribute) AYO (eval_framework.tasks.base.Language attribute) AYP (eval_framework.tasks.base.Language attribute) AYQ (eval_framework.tasks.base.Language attribute) AYR (eval_framework.tasks.base.Language attribute) AYS (eval_framework.tasks.base.Language attribute) AYT (eval_framework.tasks.base.Language attribute) AYU (eval_framework.tasks.base.Language attribute) AYX (eval_framework.tasks.base.Language attribute) AYY (eval_framework.tasks.base.Language attribute) AYZ (eval_framework.tasks.base.Language attribute) AZA (eval_framework.tasks.base.Language attribute) AZB (eval_framework.tasks.base.Language attribute) AZD (eval_framework.tasks.base.Language attribute) AZE (eval_framework.tasks.base.Language attribute) AZG (eval_framework.tasks.base.Language attribute) AZJ (eval_framework.tasks.base.Language attribute) AZM (eval_framework.tasks.base.Language attribute) AZN (eval_framework.tasks.base.Language attribute) AZO (eval_framework.tasks.base.Language attribute) AZR (eval_framework.tasks.base.Language attribute) AZT (eval_framework.tasks.base.Language attribute) AZZ (eval_framework.tasks.base.Language attribute) B BAA (eval_framework.tasks.base.Language attribute) BAB (eval_framework.tasks.base.Language attribute) BAC (eval_framework.tasks.base.Language attribute) BAE (eval_framework.tasks.base.Language attribute) BAF (eval_framework.tasks.base.Language attribute) BAG (eval_framework.tasks.base.Language attribute) BAH (eval_framework.tasks.base.Language attribute) BAJ (eval_framework.tasks.base.Language attribute) BAK (eval_framework.tasks.base.Language attribute) BAL (eval_framework.tasks.base.Language attribute) BAM (eval_framework.tasks.base.Language attribute) BAN (eval_framework.tasks.base.Language attribute) BAO (eval_framework.tasks.base.Language attribute) BAP (eval_framework.tasks.base.Language attribute) BAR (eval_framework.tasks.base.Language attribute) BAS (eval_framework.tasks.base.Language attribute) BaseConfig (class in eval_framework.base_config) BaseHFLLM (class in eval_framework.llm.huggingface) BaseLLM (class in eval_framework.llm.base) BaseLLMJudgeMetric (class in eval_framework.metrics.llm.base) BaseLoglikelihoodMetric (class in eval_framework.metrics.loglikelihood.base) BaseMetric (class in eval_framework.metrics.base) BaseTask (class in eval_framework.tasks.base) BaseVLLMModel (class in eval_framework.llm.vllm) batch_size (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) BAU (eval_framework.tasks.base.Language attribute) BAV (eval_framework.tasks.base.Language attribute) BAW (eval_framework.tasks.base.Language attribute) BAX (eval_framework.tasks.base.Language attribute) BAY (eval_framework.tasks.base.Language attribute) BAZ (eval_framework.tasks.base.Language attribute) BBA (eval_framework.tasks.base.Language attribute) BBB (eval_framework.tasks.base.Language attribute) BBC (eval_framework.tasks.base.Language attribute) BBD (eval_framework.tasks.base.Language attribute) BBE (eval_framework.tasks.base.Language attribute) BBF (eval_framework.tasks.base.Language attribute) BBG (eval_framework.tasks.base.Language attribute) BBH (eval_framework.tasks.base.Language attribute) BBI (eval_framework.tasks.base.Language attribute) BBJ (eval_framework.tasks.base.Language attribute) BBK (eval_framework.tasks.base.Language attribute) BBL (eval_framework.tasks.base.Language attribute) BBM (eval_framework.tasks.base.Language attribute) BBN (eval_framework.tasks.base.Language attribute) BBO (eval_framework.tasks.base.Language attribute) BBP (eval_framework.tasks.base.Language attribute) BBQ (eval_framework.tasks.base.Language attribute) BBR (eval_framework.tasks.base.Language attribute) BBS (eval_framework.tasks.base.Language attribute) BBT (eval_framework.tasks.base.Language attribute) BBU (eval_framework.tasks.base.Language attribute) BBV (eval_framework.tasks.base.Language attribute) BBW (eval_framework.tasks.base.Language attribute) BBX (eval_framework.tasks.base.Language attribute) BBY (eval_framework.tasks.base.Language attribute) BBZ (eval_framework.tasks.base.Language attribute) BCA (eval_framework.tasks.base.Language attribute) BCB (eval_framework.tasks.base.Language attribute) BCC (eval_framework.tasks.base.Language attribute) BCD (eval_framework.tasks.base.Language attribute) BCE (eval_framework.tasks.base.Language attribute) BCF (eval_framework.tasks.base.Language attribute) BCG (eval_framework.tasks.base.Language attribute) BCH (eval_framework.tasks.base.Language attribute) BCI (eval_framework.tasks.base.Language attribute) BCJ (eval_framework.tasks.base.Language attribute) BCK (eval_framework.tasks.base.Language attribute) BCL (eval_framework.tasks.base.Language attribute) BCM (eval_framework.tasks.base.Language attribute) BCN (eval_framework.tasks.base.Language attribute) BCO (eval_framework.tasks.base.Language attribute) BCP (eval_framework.tasks.base.Language attribute) BCQ (eval_framework.tasks.base.Language attribute) BCR (eval_framework.tasks.base.Language attribute) BCS (eval_framework.tasks.base.Language attribute) BCT (eval_framework.tasks.base.Language attribute) BCU (eval_framework.tasks.base.Language attribute) BCV (eval_framework.tasks.base.Language attribute) BCW (eval_framework.tasks.base.Language attribute) BCX (eval_framework.tasks.base.Language attribute) BCY (eval_framework.tasks.base.Language attribute) BCZ (eval_framework.tasks.base.Language attribute) BDA (eval_framework.tasks.base.Language attribute) BDB (eval_framework.tasks.base.Language attribute) BDC (eval_framework.tasks.base.Language attribute) BDD (eval_framework.tasks.base.Language attribute) BDE (eval_framework.tasks.base.Language attribute) BDF (eval_framework.tasks.base.Language attribute) BDG (eval_framework.tasks.base.Language attribute) BDH (eval_framework.tasks.base.Language attribute) BDI (eval_framework.tasks.base.Language attribute) BDJ (eval_framework.tasks.base.Language attribute) BDK (eval_framework.tasks.base.Language attribute) BDL (eval_framework.tasks.base.Language attribute) BDM (eval_framework.tasks.base.Language attribute) BDN (eval_framework.tasks.base.Language attribute) BDO (eval_framework.tasks.base.Language attribute) BDP (eval_framework.tasks.base.Language attribute) BDQ (eval_framework.tasks.base.Language attribute) BDR (eval_framework.tasks.base.Language attribute) BDS (eval_framework.tasks.base.Language attribute) BDT (eval_framework.tasks.base.Language attribute) BDU (eval_framework.tasks.base.Language attribute) BDV (eval_framework.tasks.base.Language attribute) BDW (eval_framework.tasks.base.Language attribute) BDX (eval_framework.tasks.base.Language attribute) BDY (eval_framework.tasks.base.Language attribute) BDZ (eval_framework.tasks.base.Language attribute) BEA (eval_framework.tasks.base.Language attribute) BEB (eval_framework.tasks.base.Language attribute) BEC (eval_framework.tasks.base.Language attribute) BED (eval_framework.tasks.base.Language attribute) BEE (eval_framework.tasks.base.Language attribute) BEF (eval_framework.tasks.base.Language attribute) BEG (eval_framework.tasks.base.Language attribute) BEH (eval_framework.tasks.base.Language attribute) BEI (eval_framework.tasks.base.Language attribute) BEJ (eval_framework.tasks.base.Language attribute) BEK (eval_framework.tasks.base.Language attribute) BEL (eval_framework.tasks.base.Language attribute) BELEBELE (class in eval_framework.tasks.benchmarks.belebele) BEM (eval_framework.tasks.base.Language attribute) BEN (eval_framework.tasks.base.Language attribute) benchmark_timeout (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) BEO (eval_framework.tasks.base.Language attribute) BEP (eval_framework.tasks.base.Language attribute) BEQ (eval_framework.tasks.base.Language attribute) BES (eval_framework.tasks.base.Language attribute) BET (eval_framework.tasks.base.Language attribute) BEU (eval_framework.tasks.base.Language attribute) BEV (eval_framework.tasks.base.Language attribute) BEW (eval_framework.tasks.base.Language attribute) BEX (eval_framework.tasks.base.Language attribute) BEY (eval_framework.tasks.base.Language attribute) BEZ (eval_framework.tasks.base.Language attribute) BFA (eval_framework.tasks.base.Language attribute) BFB (eval_framework.tasks.base.Language attribute) BFC (eval_framework.tasks.base.Language attribute) BFD (eval_framework.tasks.base.Language attribute) BFE (eval_framework.tasks.base.Language attribute) BFF (eval_framework.tasks.base.Language attribute) BFG (eval_framework.tasks.base.Language attribute) BFH (eval_framework.tasks.base.Language attribute) BFI (eval_framework.tasks.base.Language attribute) BFJ (eval_framework.tasks.base.Language attribute) BFK (eval_framework.tasks.base.Language attribute) BFL (eval_framework.tasks.base.Language attribute) BFM (eval_framework.tasks.base.Language attribute) BFN (eval_framework.tasks.base.Language attribute) BFO (eval_framework.tasks.base.Language attribute) BFP (eval_framework.tasks.base.Language attribute) BFQ (eval_framework.tasks.base.Language attribute) BFR (eval_framework.tasks.base.Language attribute) BFS (eval_framework.tasks.base.Language attribute) BFT (eval_framework.tasks.base.Language attribute) BFU (eval_framework.tasks.base.Language attribute) BFW (eval_framework.tasks.base.Language attribute) BFX (eval_framework.tasks.base.Language attribute) BFY (eval_framework.tasks.base.Language attribute) BFZ (eval_framework.tasks.base.Language attribute) BGA (eval_framework.tasks.base.Language attribute) BGB (eval_framework.tasks.base.Language attribute) BGC (eval_framework.tasks.base.Language attribute) BGD (eval_framework.tasks.base.Language attribute) BGE (eval_framework.tasks.base.Language attribute) BGF (eval_framework.tasks.base.Language attribute) BGG (eval_framework.tasks.base.Language attribute) BGH (eval_framework.tasks.base.Language attribute) BGI (eval_framework.tasks.base.Language attribute) BGJ (eval_framework.tasks.base.Language attribute) BGK (eval_framework.tasks.base.Language attribute) BGL (eval_framework.tasks.base.Language attribute) BGM (eval_framework.tasks.base.Language attribute) BGN (eval_framework.tasks.base.Language attribute) BGO (eval_framework.tasks.base.Language attribute) BGP (eval_framework.tasks.base.Language attribute) BGQ (eval_framework.tasks.base.Language attribute) BGR (eval_framework.tasks.base.Language attribute) BGS (eval_framework.tasks.base.Language attribute) BGT (eval_framework.tasks.base.Language attribute) BGU (eval_framework.tasks.base.Language attribute) BGV (eval_framework.tasks.base.Language attribute) BGW (eval_framework.tasks.base.Language attribute) BGX (eval_framework.tasks.base.Language attribute) BGY (eval_framework.tasks.base.Language attribute) BGZ (eval_framework.tasks.base.Language attribute) BHA (eval_framework.tasks.base.Language attribute) BHB (eval_framework.tasks.base.Language attribute) BHC (eval_framework.tasks.base.Language attribute) BHD (eval_framework.tasks.base.Language attribute) BHE (eval_framework.tasks.base.Language attribute) BHF (eval_framework.tasks.base.Language attribute) BHG (eval_framework.tasks.base.Language attribute) BHH (eval_framework.tasks.base.Language attribute) BHI (eval_framework.tasks.base.Language attribute) BHJ (eval_framework.tasks.base.Language attribute) BHK (eval_framework.tasks.base.Language attribute) BHL (eval_framework.tasks.base.Language attribute) BHM (eval_framework.tasks.base.Language attribute) BHN (eval_framework.tasks.base.Language attribute) BHO (eval_framework.tasks.base.Language attribute) BHP (eval_framework.tasks.base.Language attribute) BHQ (eval_framework.tasks.base.Language attribute) BHR (eval_framework.tasks.base.Language attribute) BHS (eval_framework.tasks.base.Language attribute) BHT (eval_framework.tasks.base.Language attribute) BHU (eval_framework.tasks.base.Language attribute) BHV (eval_framework.tasks.base.Language attribute) BHW (eval_framework.tasks.base.Language attribute) BHX (eval_framework.tasks.base.Language attribute) BHY (eval_framework.tasks.base.Language attribute) BHZ (eval_framework.tasks.base.Language attribute) BIA (eval_framework.tasks.base.Language attribute) BIB (eval_framework.tasks.base.Language attribute) BIC (eval_framework.tasks.base.Language attribute) BID (eval_framework.tasks.base.Language attribute) BIE (eval_framework.tasks.base.Language attribute) BIF (eval_framework.tasks.base.Language attribute) BIG (eval_framework.tasks.base.Language attribute) BigCodeBench (class in eval_framework.tasks.benchmarks.bigcodebench) BigCodeBenchHard (class in eval_framework.tasks.benchmarks.bigcodebench) BigCodeBenchHardInstruct (class in eval_framework.tasks.benchmarks.bigcodebench) BigCodeBenchInstruct (class in eval_framework.tasks.benchmarks.bigcodebench) BII (eval_framework.tasks.base.Language attribute) BIJ (eval_framework.tasks.base.Language attribute) BIK (eval_framework.tasks.base.Language attribute) BIL (eval_framework.tasks.base.Language attribute) BIM (eval_framework.tasks.base.Language attribute) BIN (eval_framework.tasks.base.Language attribute) BIO (eval_framework.tasks.base.Language attribute) BIP (eval_framework.tasks.base.Language attribute) BIQ (eval_framework.tasks.base.Language attribute) BIR (eval_framework.tasks.base.Language attribute) BIS (eval_framework.tasks.base.Language attribute) BIT (eval_framework.tasks.base.Language attribute) BIU (eval_framework.tasks.base.Language attribute) BIV (eval_framework.tasks.base.Language attribute) BIW (eval_framework.tasks.base.Language attribute) BIX (eval_framework.tasks.base.Language attribute) BIY (eval_framework.tasks.base.Language attribute) BIZ (eval_framework.tasks.base.Language attribute) BJA (eval_framework.tasks.base.Language attribute) BJB (eval_framework.tasks.base.Language attribute) BJC (eval_framework.tasks.base.Language attribute) BJD (eval_framework.tasks.base.Language attribute) BJE (eval_framework.tasks.base.Language attribute) BJF (eval_framework.tasks.base.Language attribute) BJG (eval_framework.tasks.base.Language attribute) BJH (eval_framework.tasks.base.Language attribute) BJI (eval_framework.tasks.base.Language attribute) BJJ (eval_framework.tasks.base.Language attribute) BJK (eval_framework.tasks.base.Language attribute) BJL (eval_framework.tasks.base.Language attribute) BJM (eval_framework.tasks.base.Language attribute) BJN (eval_framework.tasks.base.Language attribute) BJO (eval_framework.tasks.base.Language attribute) BJP (eval_framework.tasks.base.Language attribute) BJQ (eval_framework.tasks.base.Language attribute) BJR (eval_framework.tasks.base.Language attribute) BJS (eval_framework.tasks.base.Language attribute) BJT (eval_framework.tasks.base.Language attribute) BJU (eval_framework.tasks.base.Language attribute) BJV (eval_framework.tasks.base.Language attribute) BJW (eval_framework.tasks.base.Language attribute) BJX (eval_framework.tasks.base.Language attribute) BJY (eval_framework.tasks.base.Language attribute) BJZ (eval_framework.tasks.base.Language attribute) BKA (eval_framework.tasks.base.Language attribute) BKB (eval_framework.tasks.base.Language attribute) BKC (eval_framework.tasks.base.Language attribute) BKD (eval_framework.tasks.base.Language attribute) BKE (eval_framework.tasks.base.Language attribute) BKF (eval_framework.tasks.base.Language attribute) BKG (eval_framework.tasks.base.Language attribute) BKH (eval_framework.tasks.base.Language attribute) BKI (eval_framework.tasks.base.Language attribute) BKJ (eval_framework.tasks.base.Language attribute) BKK (eval_framework.tasks.base.Language attribute) BKL (eval_framework.tasks.base.Language attribute) BKM (eval_framework.tasks.base.Language attribute) BKN (eval_framework.tasks.base.Language attribute) BKO (eval_framework.tasks.base.Language attribute) BKP (eval_framework.tasks.base.Language attribute) BKQ (eval_framework.tasks.base.Language attribute) BKR (eval_framework.tasks.base.Language attribute) BKS (eval_framework.tasks.base.Language attribute) BKT (eval_framework.tasks.base.Language attribute) BKU (eval_framework.tasks.base.Language attribute) BKV (eval_framework.tasks.base.Language attribute) BKW (eval_framework.tasks.base.Language attribute) BKX (eval_framework.tasks.base.Language attribute) BKY (eval_framework.tasks.base.Language attribute) BKZ (eval_framework.tasks.base.Language attribute) BLA (eval_framework.tasks.base.Language attribute) BLB (eval_framework.tasks.base.Language attribute) BLC (eval_framework.tasks.base.Language attribute) BLD (eval_framework.tasks.base.Language attribute) BLE (eval_framework.tasks.base.Language attribute) BLEU (class in eval_framework.metrics.completion.bleu) BLF (eval_framework.tasks.base.Language attribute) BLG (eval_framework.tasks.base.Language attribute) BLH (eval_framework.tasks.base.Language attribute) BLI (eval_framework.tasks.base.Language attribute) BLJ (eval_framework.tasks.base.Language attribute) BLK (eval_framework.tasks.base.Language attribute) BLL (eval_framework.tasks.base.Language attribute) BLM (eval_framework.tasks.base.Language attribute) BLN (eval_framework.tasks.base.Language attribute) BLO (eval_framework.tasks.base.Language attribute) BLP (eval_framework.tasks.base.Language attribute) BLQ (eval_framework.tasks.base.Language attribute) BLR (eval_framework.tasks.base.Language attribute) BLS (eval_framework.tasks.base.Language attribute) BLT (eval_framework.tasks.base.Language attribute) BLU (eval_framework.tasks.base.Language attribute) BLV (eval_framework.tasks.base.Language attribute) BLW (eval_framework.tasks.base.Language attribute) BLX (eval_framework.tasks.base.Language attribute) BLY (eval_framework.tasks.base.Language attribute) BLZ (eval_framework.tasks.base.Language attribute) BMA (eval_framework.tasks.base.Language attribute) BMB (eval_framework.tasks.base.Language attribute) BMC (eval_framework.tasks.base.Language attribute) BMD (eval_framework.tasks.base.Language attribute) BME (eval_framework.tasks.base.Language attribute) BMF (eval_framework.tasks.base.Language attribute) BMG (eval_framework.tasks.base.Language attribute) BMH (eval_framework.tasks.base.Language attribute) BMI (eval_framework.tasks.base.Language attribute) BMJ (eval_framework.tasks.base.Language attribute) BMK (eval_framework.tasks.base.Language attribute) BML (eval_framework.tasks.base.Language attribute) BMM (eval_framework.tasks.base.Language attribute) BMN (eval_framework.tasks.base.Language attribute) BMO (eval_framework.tasks.base.Language attribute) BMP (eval_framework.tasks.base.Language attribute) BMQ (eval_framework.tasks.base.Language attribute) BMR (eval_framework.tasks.base.Language attribute) BMS (eval_framework.tasks.base.Language attribute) BMT (eval_framework.tasks.base.Language attribute) BMU (eval_framework.tasks.base.Language attribute) BMV (eval_framework.tasks.base.Language attribute) BMW (eval_framework.tasks.base.Language attribute) BMX (eval_framework.tasks.base.Language attribute) BMY (eval_framework.tasks.base.Language attribute) BMZ (eval_framework.tasks.base.Language attribute) BNA (eval_framework.tasks.base.Language attribute) BNB (eval_framework.tasks.base.Language attribute) BNC (eval_framework.tasks.base.Language attribute) BND (eval_framework.tasks.base.Language attribute) BNE (eval_framework.tasks.base.Language attribute) BNF (eval_framework.tasks.base.Language attribute) BNG (eval_framework.tasks.base.Language attribute) BNH (eval_framework.tasks.base.Language attribute) BNI (eval_framework.tasks.base.Language attribute) BNJ (eval_framework.tasks.base.Language attribute) BNK (eval_framework.tasks.base.Language attribute) BNL (eval_framework.tasks.base.Language attribute) BNM (eval_framework.tasks.base.Language attribute) BNN (eval_framework.tasks.base.Language attribute) BNO (eval_framework.tasks.base.Language attribute) BNP (eval_framework.tasks.base.Language attribute) BNQ (eval_framework.tasks.base.Language attribute) BNR (eval_framework.tasks.base.Language attribute) BNS (eval_framework.tasks.base.Language attribute) BNU (eval_framework.tasks.base.Language attribute) BNV (eval_framework.tasks.base.Language attribute) BNW (eval_framework.tasks.base.Language attribute) BNX (eval_framework.tasks.base.Language attribute) BNY (eval_framework.tasks.base.Language attribute) BNZ (eval_framework.tasks.base.Language attribute) BOA (eval_framework.tasks.base.Language attribute) BOB (eval_framework.tasks.base.Language attribute) BOC (eval_framework.tasks.base.Language attribute) BOD (eval_framework.tasks.base.Language attribute) BOE (eval_framework.tasks.base.Language attribute) BOF (eval_framework.tasks.base.Language attribute) BOG (eval_framework.tasks.base.Language attribute) BOH (eval_framework.tasks.base.Language attribute) BOI (eval_framework.tasks.base.Language attribute) BOJ (eval_framework.tasks.base.Language attribute) BOK (eval_framework.tasks.base.Language attribute) BOL (eval_framework.tasks.base.Language attribute) BOM (eval_framework.tasks.base.Language attribute) BON (eval_framework.tasks.base.Language attribute) BOO (eval_framework.tasks.base.Language attribute) BOP (eval_framework.tasks.base.Language attribute) BOQ (eval_framework.tasks.base.Language attribute) BOR (eval_framework.tasks.base.Language attribute) BOS (eval_framework.tasks.base.Language attribute) BOT (eval_framework.tasks.base.Language attribute) BOU (eval_framework.tasks.base.Language attribute) BOV (eval_framework.tasks.base.Language attribute) BOW (eval_framework.tasks.base.Language attribute) BOX (eval_framework.tasks.base.Language attribute) BOY (eval_framework.tasks.base.Language attribute) BOZ (eval_framework.tasks.base.Language attribute) BPA (eval_framework.tasks.base.Language attribute) BPB (eval_framework.tasks.base.Language attribute) BPC (eval_framework.tasks.base.Language attribute) BPD (eval_framework.tasks.base.Language attribute) BPE (eval_framework.tasks.base.Language attribute) BPG (eval_framework.tasks.base.Language attribute) BPH (eval_framework.tasks.base.Language attribute) BPI (eval_framework.tasks.base.Language attribute) BPJ (eval_framework.tasks.base.Language attribute) BPK (eval_framework.tasks.base.Language attribute) BPL (eval_framework.tasks.base.Language attribute) BPM (eval_framework.tasks.base.Language attribute) BPN (eval_framework.tasks.base.Language attribute) BPO (eval_framework.tasks.base.Language attribute) BPP (eval_framework.tasks.base.Language attribute) BPQ (eval_framework.tasks.base.Language attribute) BPR (eval_framework.tasks.base.Language attribute) BPS (eval_framework.tasks.base.Language attribute) BPT (eval_framework.tasks.base.Language attribute) BPU (eval_framework.tasks.base.Language attribute) BPV (eval_framework.tasks.base.Language attribute) BPW (eval_framework.tasks.base.Language attribute) BPX (eval_framework.tasks.base.Language attribute) BPY (eval_framework.tasks.base.Language attribute) BPZ (eval_framework.tasks.base.Language attribute) BQA (eval_framework.tasks.base.Language attribute) BQB (eval_framework.tasks.base.Language attribute) BQC (eval_framework.tasks.base.Language attribute) BQD (eval_framework.tasks.base.Language attribute) BQE (eval_framework.tasks.base.Language attribute) BQF (eval_framework.tasks.base.Language attribute) BQG (eval_framework.tasks.base.Language attribute) BQH (eval_framework.tasks.base.Language attribute) BQI (eval_framework.tasks.base.Language attribute) BQJ (eval_framework.tasks.base.Language attribute) BQK (eval_framework.tasks.base.Language attribute) BQL (eval_framework.tasks.base.Language attribute) BQM (eval_framework.tasks.base.Language attribute) BQN (eval_framework.tasks.base.Language attribute) BQO (eval_framework.tasks.base.Language attribute) BQP (eval_framework.tasks.base.Language attribute) BQQ (eval_framework.tasks.base.Language attribute) BQR (eval_framework.tasks.base.Language attribute) BQS (eval_framework.tasks.base.Language attribute) BQT (eval_framework.tasks.base.Language attribute) BQU (eval_framework.tasks.base.Language attribute) BQV (eval_framework.tasks.base.Language attribute) BQW (eval_framework.tasks.base.Language attribute) BQX (eval_framework.tasks.base.Language attribute) BQY (eval_framework.tasks.base.Language attribute) BQZ (eval_framework.tasks.base.Language attribute) BRA (eval_framework.tasks.base.Language attribute) BRB (eval_framework.tasks.base.Language attribute) BRC (eval_framework.tasks.base.Language attribute) BRD (eval_framework.tasks.base.Language attribute) BRE (eval_framework.tasks.base.Language attribute) BRF (eval_framework.tasks.base.Language attribute) BRG (eval_framework.tasks.base.Language attribute) BRH (eval_framework.tasks.base.Language attribute) BRI (eval_framework.tasks.base.Language attribute) BRJ (eval_framework.tasks.base.Language attribute) BRK (eval_framework.tasks.base.Language attribute) BRL (eval_framework.tasks.base.Language attribute) BRM (eval_framework.tasks.base.Language attribute) BRN (eval_framework.tasks.base.Language attribute) BRO (eval_framework.tasks.base.Language attribute) BRP (eval_framework.tasks.base.Language attribute) BRQ (eval_framework.tasks.base.Language attribute) BRR (eval_framework.tasks.base.Language attribute) BRS (eval_framework.tasks.base.Language attribute) BRT (eval_framework.tasks.base.Language attribute) BRU (eval_framework.tasks.base.Language attribute) BRV (eval_framework.tasks.base.Language attribute) BRW (eval_framework.tasks.base.Language attribute) BRX (eval_framework.tasks.base.Language attribute) BRY (eval_framework.tasks.base.Language attribute) BRZ (eval_framework.tasks.base.Language attribute) BSA (eval_framework.tasks.base.Language attribute) BSB (eval_framework.tasks.base.Language attribute) BSC (eval_framework.tasks.base.Language attribute) BSD (eval_framework.tasks.base.Language attribute) BSE (eval_framework.tasks.base.Language attribute) BSF (eval_framework.tasks.base.Language attribute) BSG (eval_framework.tasks.base.Language attribute) BSH (eval_framework.tasks.base.Language attribute) BSI (eval_framework.tasks.base.Language attribute) BSJ (eval_framework.tasks.base.Language attribute) BSK (eval_framework.tasks.base.Language attribute) BSL (eval_framework.tasks.base.Language attribute) BSM (eval_framework.tasks.base.Language attribute) BSN (eval_framework.tasks.base.Language attribute) BSO (eval_framework.tasks.base.Language attribute) BSP (eval_framework.tasks.base.Language attribute) BSQ (eval_framework.tasks.base.Language attribute) BSR (eval_framework.tasks.base.Language attribute) BSS (eval_framework.tasks.base.Language attribute) BST (eval_framework.tasks.base.Language attribute) BSU (eval_framework.tasks.base.Language attribute) BSV (eval_framework.tasks.base.Language attribute) BSW (eval_framework.tasks.base.Language attribute) BSX (eval_framework.tasks.base.Language attribute) BSY (eval_framework.tasks.base.Language attribute) BSZ (eval_framework.tasks.base.Language attribute) BTA (eval_framework.tasks.base.Language attribute) BTB (eval_framework.tasks.base.Language attribute) BTC (eval_framework.tasks.base.Language attribute) BTD (eval_framework.tasks.base.Language attribute) BTE (eval_framework.tasks.base.Language attribute) BTF (eval_framework.tasks.base.Language attribute) BTG (eval_framework.tasks.base.Language attribute) BTH (eval_framework.tasks.base.Language attribute) BTI (eval_framework.tasks.base.Language attribute) BTJ (eval_framework.tasks.base.Language attribute) BTL (eval_framework.tasks.base.Language attribute) BTM (eval_framework.tasks.base.Language attribute) BTN (eval_framework.tasks.base.Language attribute) BTO (eval_framework.tasks.base.Language attribute) BTP (eval_framework.tasks.base.Language attribute) BTQ (eval_framework.tasks.base.Language attribute) BTR (eval_framework.tasks.base.Language attribute) BTS (eval_framework.tasks.base.Language attribute) BTT (eval_framework.tasks.base.Language attribute) BTU (eval_framework.tasks.base.Language attribute) BTV (eval_framework.tasks.base.Language attribute) BTW (eval_framework.tasks.base.Language attribute) BTX (eval_framework.tasks.base.Language attribute) BTY (eval_framework.tasks.base.Language attribute) BTZ (eval_framework.tasks.base.Language attribute) BUA (eval_framework.tasks.base.Language attribute) BUB (eval_framework.tasks.base.Language attribute) BUC (eval_framework.tasks.base.Language attribute) BUD (eval_framework.tasks.base.Language attribute) BUE (eval_framework.tasks.base.Language attribute) BUF (eval_framework.tasks.base.Language attribute) BUG (eval_framework.tasks.base.Language attribute) BUH (eval_framework.tasks.base.Language attribute) BUI (eval_framework.tasks.base.Language attribute) build_redis_key_from_prompt_objs() (eval_framework.llm.vllm.BaseVLLMModel method) BUJ (eval_framework.tasks.base.Language attribute) BUK (eval_framework.tasks.base.Language attribute) BUL (eval_framework.tasks.base.Language attribute) BUM (eval_framework.tasks.base.Language attribute) BUN (eval_framework.tasks.base.Language attribute) BUO (eval_framework.tasks.base.Language attribute) BUP (eval_framework.tasks.base.Language attribute) BUQ (eval_framework.tasks.base.Language attribute) BUS (eval_framework.tasks.base.Language attribute) BUT (eval_framework.tasks.base.Language attribute) BUU (eval_framework.tasks.base.Language attribute) BUV (eval_framework.tasks.base.Language attribute) BUW (eval_framework.tasks.base.Language attribute) BUX (eval_framework.tasks.base.Language attribute) BUY (eval_framework.tasks.base.Language attribute) BUZ (eval_framework.tasks.base.Language attribute) BVA (eval_framework.tasks.base.Language attribute) BVB (eval_framework.tasks.base.Language attribute) BVC (eval_framework.tasks.base.Language attribute) BVD (eval_framework.tasks.base.Language attribute) BVE (eval_framework.tasks.base.Language attribute) BVF (eval_framework.tasks.base.Language attribute) BVG (eval_framework.tasks.base.Language attribute) BVH (eval_framework.tasks.base.Language attribute) BVI (eval_framework.tasks.base.Language attribute) BVJ (eval_framework.tasks.base.Language attribute) BVK (eval_framework.tasks.base.Language attribute) BVL (eval_framework.tasks.base.Language attribute) BVM (eval_framework.tasks.base.Language attribute) BVN (eval_framework.tasks.base.Language attribute) BVO (eval_framework.tasks.base.Language attribute) BVP (eval_framework.tasks.base.Language attribute) BVQ (eval_framework.tasks.base.Language attribute) BVR (eval_framework.tasks.base.Language attribute) BVS (eval_framework.tasks.base.Language attribute) BVT (eval_framework.tasks.base.Language attribute) BVU (eval_framework.tasks.base.Language attribute) BVV (eval_framework.tasks.base.Language attribute) BVW (eval_framework.tasks.base.Language attribute) BVX (eval_framework.tasks.base.Language attribute) BVY (eval_framework.tasks.base.Language attribute) BVZ (eval_framework.tasks.base.Language attribute) BWA (eval_framework.tasks.base.Language attribute) BWB (eval_framework.tasks.base.Language attribute) BWC (eval_framework.tasks.base.Language attribute) BWD (eval_framework.tasks.base.Language attribute) BWE (eval_framework.tasks.base.Language attribute) BWF (eval_framework.tasks.base.Language attribute) BWG (eval_framework.tasks.base.Language attribute) BWH (eval_framework.tasks.base.Language attribute) BWI (eval_framework.tasks.base.Language attribute) BWJ (eval_framework.tasks.base.Language attribute) BWK (eval_framework.tasks.base.Language attribute) BWL (eval_framework.tasks.base.Language attribute) BWM (eval_framework.tasks.base.Language attribute) BWN (eval_framework.tasks.base.Language attribute) BWO (eval_framework.tasks.base.Language attribute) BWP (eval_framework.tasks.base.Language attribute) BWQ (eval_framework.tasks.base.Language attribute) BWR (eval_framework.tasks.base.Language attribute) BWS (eval_framework.tasks.base.Language attribute) BWT (eval_framework.tasks.base.Language attribute) BWU (eval_framework.tasks.base.Language attribute) BWV (eval_framework.tasks.base.Language attribute) BWW (eval_framework.tasks.base.Language attribute) BWX (eval_framework.tasks.base.Language attribute) BWY (eval_framework.tasks.base.Language attribute) BWZ (eval_framework.tasks.base.Language attribute) BXA (eval_framework.tasks.base.Language attribute) BXB (eval_framework.tasks.base.Language attribute) BXC (eval_framework.tasks.base.Language attribute) BXD (eval_framework.tasks.base.Language attribute) BXE (eval_framework.tasks.base.Language attribute) BXF (eval_framework.tasks.base.Language attribute) BXG (eval_framework.tasks.base.Language attribute) BXH (eval_framework.tasks.base.Language attribute) BXI (eval_framework.tasks.base.Language attribute) BXJ (eval_framework.tasks.base.Language attribute) BXK (eval_framework.tasks.base.Language attribute) BXL (eval_framework.tasks.base.Language attribute) BXM (eval_framework.tasks.base.Language attribute) BXN (eval_framework.tasks.base.Language attribute) BXO (eval_framework.tasks.base.Language attribute) BXP (eval_framework.tasks.base.Language attribute) BXQ (eval_framework.tasks.base.Language attribute) BXR (eval_framework.tasks.base.Language attribute) BXS (eval_framework.tasks.base.Language attribute) BXT (eval_framework.tasks.base.Language attribute) BXU (eval_framework.tasks.base.Language attribute) BXV (eval_framework.tasks.base.Language attribute) BXW (eval_framework.tasks.base.Language attribute) BXX (eval_framework.tasks.base.Language attribute) BXZ (eval_framework.tasks.base.Language attribute) BYA (eval_framework.tasks.base.Language attribute) BYB (eval_framework.tasks.base.Language attribute) BYC (eval_framework.tasks.base.Language attribute) BYD (eval_framework.tasks.base.Language attribute) BYE (eval_framework.tasks.base.Language attribute) BYF (eval_framework.tasks.base.Language attribute) BYG (eval_framework.tasks.base.Language attribute) BYH (eval_framework.tasks.base.Language attribute) BYI (eval_framework.tasks.base.Language attribute) BYJ (eval_framework.tasks.base.Language attribute) BYK (eval_framework.tasks.base.Language attribute) BYL (eval_framework.tasks.base.Language attribute) BYM (eval_framework.tasks.base.Language attribute) BYN (eval_framework.tasks.base.Language attribute) BYO (eval_framework.tasks.base.Language attribute) BYP (eval_framework.tasks.base.Language attribute) BYQ (eval_framework.tasks.base.Language attribute) BYR (eval_framework.tasks.base.Language attribute) BYS (eval_framework.tasks.base.Language attribute) BYT (eval_framework.tasks.base.Language attribute) BYTES_PER_TOKEN (eval_framework.llm.aleph_alpha.AlephAlphaAPIModel attribute) (eval_framework.llm.huggingface.BaseHFLLM attribute) (eval_framework.llm.openai.OpenAIModel attribute) (eval_framework.llm.vllm.BaseVLLMModel attribute) BytesCompletion (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) BytesLoglikelihood (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) BYU (eval_framework.tasks.base.Language attribute) BYV (eval_framework.tasks.base.Language attribute) BYW (eval_framework.tasks.base.Language attribute) BYX (eval_framework.tasks.base.Language attribute) BYY (eval_framework.tasks.base.Language attribute) BYZ (eval_framework.tasks.base.Language attribute) BZA (eval_framework.tasks.base.Language attribute) BZB (eval_framework.tasks.base.Language attribute) BZC (eval_framework.tasks.base.Language attribute) BZD (eval_framework.tasks.base.Language attribute) BZE (eval_framework.tasks.base.Language attribute) BZF (eval_framework.tasks.base.Language attribute) BZG (eval_framework.tasks.base.Language attribute) BZH (eval_framework.tasks.base.Language attribute) BZI (eval_framework.tasks.base.Language attribute) BZJ (eval_framework.tasks.base.Language attribute) BZK (eval_framework.tasks.base.Language attribute) BZL (eval_framework.tasks.base.Language attribute) BZM (eval_framework.tasks.base.Language attribute) BZN (eval_framework.tasks.base.Language attribute) BZO (eval_framework.tasks.base.Language attribute) BZP (eval_framework.tasks.base.Language attribute) BZQ (eval_framework.tasks.base.Language attribute) BZR (eval_framework.tasks.base.Language attribute) BZS (eval_framework.tasks.base.Language attribute) BZT (eval_framework.tasks.base.Language attribute) BZU (eval_framework.tasks.base.Language attribute) BZV (eval_framework.tasks.base.Language attribute) BZW (eval_framework.tasks.base.Language attribute) BZX (eval_framework.tasks.base.Language attribute) BZY (eval_framework.tasks.base.Language attribute) BZZ (eval_framework.tasks.base.Language attribute) C CAA (eval_framework.tasks.base.Language attribute) CAB (eval_framework.tasks.base.Language attribute) CAC (eval_framework.tasks.base.Language attribute) CAD (eval_framework.tasks.base.Language attribute) CAE (eval_framework.tasks.base.Language attribute) CAF (eval_framework.tasks.base.Language attribute) CAG (eval_framework.tasks.base.Language attribute) CAH (eval_framework.tasks.base.Language attribute) CAJ (eval_framework.tasks.base.Language attribute) CAK (eval_framework.tasks.base.Language attribute) CAL (eval_framework.tasks.base.Language attribute) calculate() (eval_framework.metrics.base.BaseMetric method) (eval_framework.metrics.completion.accuracy_completion.AccuracyCompletion method) (eval_framework.metrics.completion.aidanbench.AidanBenchMetric method) (eval_framework.metrics.completion.bleu.BLEU method) (eval_framework.metrics.completion.bleu.LINEWISE_BLEU method) (eval_framework.metrics.completion.bleu.ResponseToOriginalBLEU method) (eval_framework.metrics.completion.chrf.CHRF method) (eval_framework.metrics.completion.chrf.LINEWISE_CHRF method) (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion method) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOne method) (eval_framework.metrics.completion.comet.COMET method) (eval_framework.metrics.completion.concordance_index.ConcordanceIndex method) (eval_framework.metrics.completion.csv_format.CSVFormat method) (eval_framework.metrics.completion.cwe_accuracy.CWEAccuracy method) (eval_framework.metrics.completion.exponential_similarity.ExponentialSimilarity method) (eval_framework.metrics.completion.f1.F1 method) (eval_framework.metrics.completion.format_checker.CheckJsonFormat method) (eval_framework.metrics.completion.format_checker.CheckPostScriptFormat method) (eval_framework.metrics.completion.grid_difference.GridDifference method) (eval_framework.metrics.completion.ifeval.IFEvalMetric method) (eval_framework.metrics.completion.json_format.JsonFormat method) (eval_framework.metrics.completion.language_checker.GermanCompletionChecker method) (eval_framework.metrics.completion.language_checker.LanguageChecker method) (eval_framework.metrics.completion.language_checker.LanguageConsistencyChecker method) (eval_framework.metrics.completion.language_checker.LanguageRawConsistencyChecker method) (eval_framework.metrics.completion.length_control.LengthControl method) (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion method) (eval_framework.metrics.completion.niah_accuracy.NIAHAccuracy method) (eval_framework.metrics.completion.placeholder_checker.PlaceholderChecker method) (eval_framework.metrics.completion.repetition.WordRepetition method) (eval_framework.metrics.completion.rouge_1.ROUGE_1 method) (eval_framework.metrics.completion.rouge_2.ROUGE_2 method) (eval_framework.metrics.completion.rouge_geometric_mean.ROUGE_GEOMETRIC_MEAN method) (eval_framework.metrics.completion.rouge_l.ROUGE_L method) (eval_framework.metrics.completion.struct_eval_metrics.RenderableStructMetric method) (eval_framework.metrics.completion.struct_eval_metrics.StructMetric method) (eval_framework.metrics.completion.ter.LINEWISE_TER method) (eval_framework.metrics.completion.ter.TER method) (eval_framework.metrics.completion.text_counter.ParagraphCounter method) (eval_framework.metrics.completion.text_counter.ResponseToOriginalLengthRatio method) (eval_framework.metrics.completion.text_counter.SentenceCounter method) (eval_framework.metrics.completion.text_counter.WordCounter method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesCompletion method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesLoglikelihood method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsCompletion method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsLoglikelihood method) (eval_framework.metrics.llm.llm_judge_chatbot_style.LLMJudgeChatbotStyle method) (eval_framework.metrics.llm.llm_judge_coherence.LLMJudgeCoherence method) (eval_framework.metrics.llm.llm_judge_completion_accuracy.LLMJudgeCompletionAccuracy method) (eval_framework.metrics.llm.llm_judge_conciseness.LLMJudgeConciseness method) (eval_framework.metrics.llm.llm_judge_contains_names.LLMJudgeAvoidsNames method) (eval_framework.metrics.llm.llm_judge_format_correctness.LLMJudgeFormatCorrectness method) (eval_framework.metrics.llm.llm_judge_instruction.LLMJudgeInstruction method) (eval_framework.metrics.llm.llm_judge_mtbench_pair.MTBenchJudgePair method) (eval_framework.metrics.llm.llm_judge_mtbench_single.MTBenchJudgeSingle method) (eval_framework.metrics.llm.llm_judge_refusal.LLMJudgeRefusal method) (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) (eval_framework.metrics.llm.llm_judge_world_knowledge.LLMJudgeWorldKnowledge method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyLoglikelihood method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyNormLoglikelihood method) (eval_framework.metrics.loglikelihood.confidence_weighted_accuracy.ConfidenceWeightedAccuracy method) (eval_framework.metrics.loglikelihood.dcs.DistributionalCorrectnessScore method) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMass method) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMassNorm method) (eval_framework.metrics.loglikelihood.ternary.TernaryScore method) calculate_concordance_index() (in module eval_framework.metrics.completion.concordance_index) calculate_exponential_similarity() (in module eval_framework.metrics.completion.exponential_similarity) calculate_f1() (in module eval_framework.metrics.completion.f1) calculate_score() (eval_framework.metrics.completion.grid_difference.GridDifference method) CallableSerializer (class in eval_framework.tasks.utils) CAM (eval_framework.tasks.base.Language attribute) CAN (eval_framework.tasks.base.Language attribute) candidate_is_a (eval_framework.metrics.llm.llm_judge_mtbench_pair.PromptToJudge attribute) CAO (eval_framework.tasks.base.Language attribute) CAP (eval_framework.tasks.base.Language attribute) CAQ (eval_framework.tasks.base.Language attribute) CAR (eval_framework.tasks.base.Language attribute) CAS (eval_framework.tasks.base.Language attribute) CASEHOLD (class in eval_framework.tasks.benchmarks.casehold) CAT (eval_framework.tasks.base.Language attribute) category (eval_framework.metrics.llm.llm_judge_mtbench_pair.MTBenchJudgePairMetricContext attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.MTBenchJudgeSingleMetricContext attribute) CAV (eval_framework.tasks.base.Language attribute) CAW (eval_framework.tasks.base.Language attribute) CAX (eval_framework.tasks.base.Language attribute) CAY (eval_framework.tasks.base.Language attribute) CAZ (eval_framework.tasks.base.Language attribute) CBB (eval_framework.tasks.base.Language attribute) CBC (eval_framework.tasks.base.Language attribute) CBD (eval_framework.tasks.base.Language attribute) CBE (eval_framework.tasks.base.Language attribute) CBG (eval_framework.tasks.base.Language attribute) CBH (eval_framework.tasks.base.Language attribute) CBI (eval_framework.tasks.base.Language attribute) CBJ (eval_framework.tasks.base.Language attribute) CBK (eval_framework.tasks.base.Language attribute) CBL (eval_framework.tasks.base.Language attribute) CBM (eval_framework.tasks.base.Language attribute) CBN (eval_framework.tasks.base.Language attribute) CBO (eval_framework.tasks.base.Language attribute) CBQ (eval_framework.tasks.base.Language attribute) CBR (eval_framework.tasks.base.Language attribute) CBS (eval_framework.tasks.base.Language attribute) CBT (eval_framework.tasks.base.Language attribute) CBU (eval_framework.tasks.base.Language attribute) CBV (eval_framework.tasks.base.Language attribute) CBW (eval_framework.tasks.base.Language attribute) CBY (eval_framework.tasks.base.Language attribute) CCA (eval_framework.tasks.base.Language attribute) CCC (eval_framework.tasks.base.Language attribute) CCD (eval_framework.tasks.base.Language attribute) CCE (eval_framework.tasks.base.Language attribute) CCG (eval_framework.tasks.base.Language attribute) CCH (eval_framework.tasks.base.Language attribute) CCJ (eval_framework.tasks.base.Language attribute) CCL (eval_framework.tasks.base.Language attribute) CCM (eval_framework.tasks.base.Language attribute) CCO (eval_framework.tasks.base.Language attribute) CCP (eval_framework.tasks.base.Language attribute) CCQ (eval_framework.tasks.base.Language attribute) CCR (eval_framework.tasks.base.Language attribute) CCX (eval_framework.tasks.base.Language attribute) CCY (eval_framework.tasks.base.Language attribute) CDA (eval_framework.tasks.base.Language attribute) CDE (eval_framework.tasks.base.Language attribute) CDF (eval_framework.tasks.base.Language attribute) CDG (eval_framework.tasks.base.Language attribute) CDH (eval_framework.tasks.base.Language attribute) CDI (eval_framework.tasks.base.Language attribute) CDJ (eval_framework.tasks.base.Language attribute) CDM (eval_framework.tasks.base.Language attribute) CDN (eval_framework.tasks.base.Language attribute) CDO (eval_framework.tasks.base.Language attribute) CDR (eval_framework.tasks.base.Language attribute) CDS (eval_framework.tasks.base.Language attribute) CDY (eval_framework.tasks.base.Language attribute) CDZ (eval_framework.tasks.base.Language attribute) CEA (eval_framework.tasks.base.Language attribute) CEB (eval_framework.tasks.base.Language attribute) CEG (eval_framework.tasks.base.Language attribute) CEK (eval_framework.tasks.base.Language attribute) CEN (eval_framework.tasks.base.Language attribute) CES (eval_framework.tasks.base.Language attribute) CET (eval_framework.tasks.base.Language attribute) CEY (eval_framework.tasks.base.Language attribute) CFA (eval_framework.tasks.base.Language attribute) CFD (eval_framework.tasks.base.Language attribute) CFG (eval_framework.tasks.base.Language attribute) CFM (eval_framework.tasks.base.Language attribute) CGA (eval_framework.tasks.base.Language attribute) CGC (eval_framework.tasks.base.Language attribute) CGG (eval_framework.tasks.base.Language attribute) CGK (eval_framework.tasks.base.Language attribute) CHA (eval_framework.tasks.base.Language attribute) chat_template (eval_framework.llm.vllm.HFTokenizerProtocol property) (eval_framework.llm.vllm.VLLMTokenizer property) (eval_framework.llm.vllm.VLLMTokenizerAPI property) CHB (eval_framework.tasks.base.Language attribute) CHC (eval_framework.tasks.base.Language attribute) CHD (eval_framework.tasks.base.Language attribute) CHE (eval_framework.tasks.base.Language attribute) check_for_equation() (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion method) CheckJsonFormat (class in eval_framework.metrics.completion.format_checker) CheckPostScriptFormat (class in eval_framework.metrics.completion.format_checker) ChemBench (class in eval_framework.tasks.benchmarks.chembench) CHF (eval_framework.tasks.base.Language attribute) CHG (eval_framework.tasks.base.Language attribute) CHH (eval_framework.tasks.base.Language attribute) CHJ (eval_framework.tasks.base.Language attribute) CHK (eval_framework.tasks.base.Language attribute) CHL (eval_framework.tasks.base.Language attribute) CHM (eval_framework.tasks.base.Language attribute) CHN (eval_framework.tasks.base.Language attribute) CHO (eval_framework.tasks.base.Language attribute) CHP (eval_framework.tasks.base.Language attribute) CHQ (eval_framework.tasks.base.Language attribute) CHR (eval_framework.tasks.base.Language attribute) CHRF (class in eval_framework.metrics.completion.chrf) CHS (eval_framework.tasks.base.Language attribute) CHT (eval_framework.tasks.base.Language attribute) CHU (eval_framework.tasks.base.Language attribute) CHV (eval_framework.tasks.base.Language attribute) CHW (eval_framework.tasks.base.Language attribute) CHX (eval_framework.tasks.base.Language attribute) CHY (eval_framework.tasks.base.Language attribute) CHZ (eval_framework.tasks.base.Language attribute) CIA (eval_framework.tasks.base.Language attribute) CIB (eval_framework.tasks.base.Language attribute) CIC (eval_framework.tasks.base.Language attribute) CID (eval_framework.tasks.base.Language attribute) CIE (eval_framework.tasks.base.Language attribute) CIH (eval_framework.tasks.base.Language attribute) CIK (eval_framework.tasks.base.Language attribute) CIM (eval_framework.tasks.base.Language attribute) CIN (eval_framework.tasks.base.Language attribute) CIP (eval_framework.tasks.base.Language attribute) CIR (eval_framework.tasks.base.Language attribute) CIT (eval_framework.tasks.base.Language attribute) CIW (eval_framework.tasks.base.Language attribute) CIY (eval_framework.tasks.base.Language attribute) CJA (eval_framework.tasks.base.Language attribute) CJE (eval_framework.tasks.base.Language attribute) CJH (eval_framework.tasks.base.Language attribute) CJI (eval_framework.tasks.base.Language attribute) CJK (eval_framework.tasks.base.Language attribute) CJM (eval_framework.tasks.base.Language attribute) CJN (eval_framework.tasks.base.Language attribute) CJO (eval_framework.tasks.base.Language attribute) CJP (eval_framework.tasks.base.Language attribute) CJR (eval_framework.tasks.base.Language attribute) CJS (eval_framework.tasks.base.Language attribute) CJV (eval_framework.tasks.base.Language attribute) CJY (eval_framework.tasks.base.Language attribute) CKA (eval_framework.tasks.base.Language attribute) CKB (eval_framework.tasks.base.Language attribute) CKC (eval_framework.tasks.base.Language attribute) CKD (eval_framework.tasks.base.Language attribute) CKE (eval_framework.tasks.base.Language attribute) CKF (eval_framework.tasks.base.Language attribute) CKH (eval_framework.tasks.base.Language attribute) CKI (eval_framework.tasks.base.Language attribute) CKJ (eval_framework.tasks.base.Language attribute) CKK (eval_framework.tasks.base.Language attribute) CKL (eval_framework.tasks.base.Language attribute) CKM (eval_framework.tasks.base.Language attribute) CKN (eval_framework.tasks.base.Language attribute) CKO (eval_framework.tasks.base.Language attribute) CKQ (eval_framework.tasks.base.Language attribute) CKR (eval_framework.tasks.base.Language attribute) CKS (eval_framework.tasks.base.Language attribute) CKT (eval_framework.tasks.base.Language attribute) CKU (eval_framework.tasks.base.Language attribute) CKV (eval_framework.tasks.base.Language attribute) CKW (eval_framework.tasks.base.Language attribute) CKX (eval_framework.tasks.base.Language attribute) CKY (eval_framework.tasks.base.Language attribute) CKZ (eval_framework.tasks.base.Language attribute) CLA (eval_framework.tasks.base.Language attribute) classproperty (class in eval_framework.metrics.base) CLC (eval_framework.tasks.base.Language attribute) CLD (eval_framework.tasks.base.Language attribute) CLE (eval_framework.tasks.base.Language attribute) clean_text() (in module eval_framework.metrics.completion.niah_accuracy) CLH (eval_framework.tasks.base.Language attribute) CLI (eval_framework.tasks.base.Language attribute) CLJ (eval_framework.tasks.base.Language attribute) CLK (eval_framework.tasks.base.Language attribute) CLL (eval_framework.tasks.base.Language attribute) CLM (eval_framework.tasks.base.Language attribute) CLO (eval_framework.tasks.base.Language attribute) CLS (eval_framework.tasks.base.Language attribute) CLT (eval_framework.tasks.base.Language attribute) CLU (eval_framework.tasks.base.Language attribute) CLW (eval_framework.tasks.base.Language attribute) CLY (eval_framework.tasks.base.Language attribute) CMA (eval_framework.tasks.base.Language attribute) CME (eval_framework.tasks.base.Language attribute) CMG (eval_framework.tasks.base.Language attribute) CMI (eval_framework.tasks.base.Language attribute) CMK (eval_framework.tasks.base.Language attribute) CML (eval_framework.tasks.base.Language attribute) CMM (eval_framework.tasks.base.Language attribute) CMN (eval_framework.tasks.base.Language attribute) CMO (eval_framework.tasks.base.Language attribute) CMR (eval_framework.tasks.base.Language attribute) CMS (eval_framework.tasks.base.Language attribute) CMT (eval_framework.tasks.base.Language attribute) CNA (eval_framework.tasks.base.Language attribute) CNB (eval_framework.tasks.base.Language attribute) CNC (eval_framework.tasks.base.Language attribute) CNG (eval_framework.tasks.base.Language attribute) CNH (eval_framework.tasks.base.Language attribute) CNI (eval_framework.tasks.base.Language attribute) CNK (eval_framework.tasks.base.Language attribute) CNL (eval_framework.tasks.base.Language attribute) CNM (eval_framework.tasks.base.Language attribute) CNO (eval_framework.tasks.base.Language attribute) CNP (eval_framework.tasks.base.Language attribute) CNQ (eval_framework.tasks.base.Language attribute) CNR (eval_framework.tasks.base.Language attribute) CNS (eval_framework.tasks.base.Language attribute) CNT (eval_framework.tasks.base.Language attribute) CNU (eval_framework.tasks.base.Language attribute) CNW (eval_framework.tasks.base.Language attribute) CNX (eval_framework.tasks.base.Language attribute) COA (eval_framework.tasks.base.Language attribute) COB (eval_framework.tasks.base.Language attribute) COC (eval_framework.tasks.base.Language attribute) COD (eval_framework.tasks.base.Language attribute) code_execution_trace (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) code_prompt (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) CodeCompletionAssertion (class in eval_framework.metrics.completion.code_assertion) CodeExecutionBaseContext (class in eval_framework.metrics.completion.code_execution_pass_at_one) CodeExecutionPassAtOne (class in eval_framework.metrics.completion.code_execution_pass_at_one) CodeExecutionPassAtOneContext (class in eval_framework.metrics.completion.code_execution_pass_at_one) COE (eval_framework.tasks.base.Language attribute) COF (eval_framework.tasks.base.Language attribute) COG (eval_framework.tasks.base.Language attribute) COH (eval_framework.tasks.base.Language attribute) COJ (eval_framework.tasks.base.Language attribute) COK (eval_framework.tasks.base.Language attribute) COL (eval_framework.tasks.base.Language attribute) COM (eval_framework.tasks.base.Language attribute) COMET (class in eval_framework.metrics.completion.comet) comparison (eval_framework.metrics.completion.text_counter.ParagraphCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.SentenceCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.WordCounterMetricContext attribute) comparison_type (eval_framework.metrics.llm.llm_judge_mtbench_pair.PromptToJudge attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.PromptToJudge attribute) COMPLETION (eval_framework.tasks.base.ResponseType attribute) COMPLETION_PREFIX (eval_framework.tasks.benchmarks.wmt.WMT_INSTRUCT attribute) CON (eval_framework.tasks.base.Language attribute) ConcordanceIndex (class in eval_framework.metrics.completion.concordance_index) ConfidenceWeightedAccuracy (class in eval_framework.metrics.loglikelihood.confidence_weighted_accuracy) connect_to_mysql() (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) connect_to_postgres() (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) context (eval_framework.tasks.base.Sample attribute) convert_choice() (eval_framework.tasks.benchmarks.copa.COPA method) COO (eval_framework.tasks.base.Language attribute) COP (eval_framework.tasks.base.Language attribute) COPA (class in eval_framework.tasks.benchmarks.copa) COPA_IDK (class in eval_framework.tasks.benchmarks.copa) COQ (eval_framework.tasks.base.Language attribute) COR (eval_framework.tasks.base.Language attribute) COS (eval_framework.tasks.base.Language attribute) COT (eval_framework.tasks.base.Language attribute) COU (eval_framework.tasks.base.Language attribute) count_differences() (eval_framework.metrics.completion.grid_difference.GridDifference method) count_result_columns() (in module eval_framework.metrics.llm.llm_judge_sql) count_tokens() (eval_framework.llm.huggingface.BaseHFLLM method) (eval_framework.llm.vllm.BaseVLLMModel method) COV (eval_framework.tasks.base.Language attribute) COW (eval_framework.tasks.base.Language attribute) COX (eval_framework.tasks.base.Language attribute) COY (eval_framework.tasks.base.Language attribute) COZ (eval_framework.tasks.base.Language attribute) CPA (eval_framework.tasks.base.Language attribute) CPB (eval_framework.tasks.base.Language attribute) CPC (eval_framework.tasks.base.Language attribute) CPG (eval_framework.tasks.base.Language attribute) CPI (eval_framework.tasks.base.Language attribute) CPN (eval_framework.tasks.base.Language attribute) CPO (eval_framework.tasks.base.Language attribute) CPS (eval_framework.tasks.base.Language attribute) CPU (eval_framework.tasks.base.Language attribute) CPX (eval_framework.tasks.base.Language attribute) CPY (eval_framework.tasks.base.Language attribute) CQD (eval_framework.tasks.base.Language attribute) CQU (eval_framework.tasks.base.Language attribute) CRA (eval_framework.tasks.base.Language attribute) CRB (eval_framework.tasks.base.Language attribute) CRC (eval_framework.tasks.base.Language attribute) CRD (eval_framework.tasks.base.Language attribute) CRE (eval_framework.tasks.base.Language attribute) create_perturbation_class() (in module eval_framework.tasks.perturbation) CRF (eval_framework.tasks.base.Language attribute) CRG (eval_framework.tasks.base.Language attribute) CRH (eval_framework.tasks.base.Language attribute) CRI (eval_framework.tasks.base.Language attribute) CRJ (eval_framework.tasks.base.Language attribute) CRK (eval_framework.tasks.base.Language attribute) CRL (eval_framework.tasks.base.Language attribute) CRM (eval_framework.tasks.base.Language attribute) CRN (eval_framework.tasks.base.Language attribute) CRO (eval_framework.tasks.base.Language attribute) CRQ (eval_framework.tasks.base.Language attribute) CRR (eval_framework.tasks.base.Language attribute) CRS (eval_framework.tasks.base.Language attribute) CRT (eval_framework.tasks.base.Language attribute) CRU (eval_framework.tasks.base.Language attribute) CRV (eval_framework.tasks.base.Language attribute) CRW (eval_framework.tasks.base.Language attribute) CRX (eval_framework.tasks.base.Language attribute) CRY (eval_framework.tasks.base.Language attribute) CRZ (eval_framework.tasks.base.Language attribute) CSA (eval_framework.tasks.base.Language attribute) CSB (eval_framework.tasks.base.Language attribute) CSC (eval_framework.tasks.base.Language attribute) CSD (eval_framework.tasks.base.Language attribute) CSE (eval_framework.tasks.base.Language attribute) CSF (eval_framework.tasks.base.Language attribute) CSG (eval_framework.tasks.base.Language attribute) CSH (eval_framework.tasks.base.Language attribute) CSI (eval_framework.tasks.base.Language attribute) CSJ (eval_framework.tasks.base.Language attribute) CSK (eval_framework.tasks.base.Language attribute) CSL (eval_framework.tasks.base.Language attribute) CSM (eval_framework.tasks.base.Language attribute) CSN (eval_framework.tasks.base.Language attribute) CSO (eval_framework.tasks.base.Language attribute) CSP (eval_framework.tasks.base.Language attribute) CSQ (eval_framework.tasks.base.Language attribute) CSR (eval_framework.tasks.base.Language attribute) CSS (eval_framework.tasks.base.Language attribute) CST (eval_framework.tasks.base.Language attribute) CSV (eval_framework.tasks.base.Language attribute) CSVFormat (class in eval_framework.metrics.completion.csv_format) CSVFormatEvaluation (class in eval_framework.metrics.completion.csv_format) CSW (eval_framework.tasks.base.Language attribute) CSX (eval_framework.tasks.base.Language attribute) CSY (eval_framework.tasks.base.Language attribute) CSZ (eval_framework.tasks.base.Language attribute) CTA (eval_framework.tasks.base.Language attribute) CTC (eval_framework.tasks.base.Language attribute) CTD (eval_framework.tasks.base.Language attribute) CTE (eval_framework.tasks.base.Language attribute) CTG (eval_framework.tasks.base.Language attribute) CTH (eval_framework.tasks.base.Language attribute) CTI (eval_framework.tasks.base.Language attribute) CTL (eval_framework.tasks.base.Language attribute) CTM (eval_framework.tasks.base.Language attribute) CTN (eval_framework.tasks.base.Language attribute) CTO (eval_framework.tasks.base.Language attribute) CTP (eval_framework.tasks.base.Language attribute) CTS (eval_framework.tasks.base.Language attribute) CTT (eval_framework.tasks.base.Language attribute) CTU (eval_framework.tasks.base.Language attribute) CTY (eval_framework.tasks.base.Language attribute) CTZ (eval_framework.tasks.base.Language attribute) CUA (eval_framework.tasks.base.Language attribute) CUB (eval_framework.tasks.base.Language attribute) CUC (eval_framework.tasks.base.Language attribute) CUE_PREFIX (eval_framework.tasks.benchmarks.humaneval.HumanEvalInstruct attribute) CUG (eval_framework.tasks.base.Language attribute) CUH (eval_framework.tasks.base.Language attribute) CUI (eval_framework.tasks.base.Language attribute) CUJ (eval_framework.tasks.base.Language attribute) CUK (eval_framework.tasks.base.Language attribute) CUL (eval_framework.tasks.base.Language attribute) CUM (eval_framework.tasks.base.Language attribute) CUN (eval_framework.tasks.base.Language attribute) CUO (eval_framework.tasks.base.Language attribute) CUP (eval_framework.tasks.base.Language attribute) CUQ (eval_framework.tasks.base.Language attribute) CUR (eval_framework.tasks.base.Language attribute) CUT (eval_framework.tasks.base.Language attribute) CUU (eval_framework.tasks.base.Language attribute) CUV (eval_framework.tasks.base.Language attribute) CUW (eval_framework.tasks.base.Language attribute) CUX (eval_framework.tasks.base.Language attribute) CUY (eval_framework.tasks.base.Language attribute) CVG (eval_framework.tasks.base.Language attribute) CVN (eval_framework.tasks.base.Language attribute) CWA (eval_framework.tasks.base.Language attribute) CWB (eval_framework.tasks.base.Language attribute) CWD (eval_framework.tasks.base.Language attribute) CWE (eval_framework.tasks.base.Language attribute) CWEAccuracy (class in eval_framework.metrics.completion.cwe_accuracy) CWG (eval_framework.tasks.base.Language attribute) CWT (eval_framework.tasks.base.Language attribute) CXH (eval_framework.tasks.base.Language attribute) CYA (eval_framework.tasks.base.Language attribute) CYB (eval_framework.tasks.base.Language attribute) CYM (eval_framework.tasks.base.Language attribute) CYO (eval_framework.tasks.base.Language attribute) CZH (eval_framework.tasks.base.Language attribute) CZK (eval_framework.tasks.base.Language attribute) CZN (eval_framework.tasks.base.Language attribute) CZO (eval_framework.tasks.base.Language attribute) CZT (eval_framework.tasks.base.Language attribute) D DAA (eval_framework.tasks.base.Language attribute) DAC (eval_framework.tasks.base.Language attribute) DAD (eval_framework.tasks.base.Language attribute) DAE (eval_framework.tasks.base.Language attribute) DAF (eval_framework.tasks.base.Language attribute) DAG (eval_framework.tasks.base.Language attribute) DAH (eval_framework.tasks.base.Language attribute) DAI (eval_framework.tasks.base.Language attribute) DAJ (eval_framework.tasks.base.Language attribute) DAK (eval_framework.tasks.base.Language attribute) DAL (eval_framework.tasks.base.Language attribute) DAM (eval_framework.tasks.base.Language attribute) DAN (eval_framework.tasks.base.Language attribute) DAO (eval_framework.tasks.base.Language attribute) DAP (eval_framework.tasks.base.Language attribute) DAQ (eval_framework.tasks.base.Language attribute) DAR (eval_framework.tasks.base.Language attribute) DAS (eval_framework.tasks.base.Language attribute) DAT (eval_framework.tasks.base.Language attribute) DATASET_PATH (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBenchHard attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalDe attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalFiSv attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2025 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.TRUTHFULQA_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.TRUTHFULQA_EU20_FR attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winox.WINOX attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.wmt.WMT14 attribute) (eval_framework.tasks.benchmarks.wmt.WMT14_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT16 attribute) (eval_framework.tasks.benchmarks.wmt.WMT16_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT20 attribute) (eval_framework.tasks.benchmarks.wmt.WMT20_INSTRUCT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_COMPLETION attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) DAU (eval_framework.tasks.base.Language attribute) DAV (eval_framework.tasks.base.Language attribute) DAW (eval_framework.tasks.base.Language attribute) DAX (eval_framework.tasks.base.Language attribute) DAZ (eval_framework.tasks.base.Language attribute) db_schema (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSqlMetricContext attribute) DBA (eval_framework.tasks.base.Language attribute) DBB (eval_framework.tasks.base.Language attribute) DBD (eval_framework.tasks.base.Language attribute) DBE (eval_framework.tasks.base.Language attribute) DBF (eval_framework.tasks.base.Language attribute) DBG (eval_framework.tasks.base.Language attribute) DBI (eval_framework.tasks.base.Language attribute) DBJ (eval_framework.tasks.base.Language attribute) DBL (eval_framework.tasks.base.Language attribute) DBM (eval_framework.tasks.base.Language attribute) DBN (eval_framework.tasks.base.Language attribute) DBO (eval_framework.tasks.base.Language attribute) DBP (eval_framework.tasks.base.Language attribute) DBQ (eval_framework.tasks.base.Language attribute) DBR (eval_framework.tasks.base.Language attribute) DBT (eval_framework.tasks.base.Language attribute) DBU (eval_framework.tasks.base.Language attribute) DBV (eval_framework.tasks.base.Language attribute) DBW (eval_framework.tasks.base.Language attribute) DBY (eval_framework.tasks.base.Language attribute) DCC (eval_framework.tasks.base.Language attribute) DCR (eval_framework.tasks.base.Language attribute) DDA (eval_framework.tasks.base.Language attribute) DDD (eval_framework.tasks.base.Language attribute) DDE (eval_framework.tasks.base.Language attribute) DDG (eval_framework.tasks.base.Language attribute) DDI (eval_framework.tasks.base.Language attribute) DDJ (eval_framework.tasks.base.Language attribute) DDN (eval_framework.tasks.base.Language attribute) DDO (eval_framework.tasks.base.Language attribute) DDR (eval_framework.tasks.base.Language attribute) DDS (eval_framework.tasks.base.Language attribute) DDW (eval_framework.tasks.base.Language attribute) DEC (eval_framework.tasks.base.Language attribute) decode() (eval_framework.llm.vllm.HFTokenizerProtocol method) (eval_framework.llm.vllm.VLLMTokenizer method) (eval_framework.tasks.utils.CallableSerializer static method) DecodingError DED (eval_framework.tasks.base.Language attribute) DEE (eval_framework.tasks.base.Language attribute) Deepseek_chat (class in eval_framework.llm.openai) Deepseek_chat_with_formatter (class in eval_framework.llm.openai) Deepseek_reasoner (class in eval_framework.llm.openai) DeepseekModel (class in eval_framework.llm.openai) DEF (eval_framework.tasks.base.Language attribute) DEFAULT_FORMATTER (eval_framework.llm.aleph_alpha.AlephAlphaAPIModel attribute) (eval_framework.llm.aleph_alpha.Llama31_8B_Instruct_API attribute) (eval_framework.llm.huggingface.BaseHFLLM attribute) (eval_framework.llm.huggingface.Pythia410m attribute) (eval_framework.llm.huggingface.Qwen3_0_6B attribute) (eval_framework.llm.huggingface.SmolLM135M attribute) (eval_framework.llm.huggingface.Smollm135MInstruct attribute) (eval_framework.llm.openai.Deepseek_chat_with_formatter attribute) (eval_framework.llm.openai.OpenAI_davinci_002 attribute) (eval_framework.llm.openai.OpenAI_gpt_4o_mini_with_ConcatFormatter attribute) (eval_framework.llm.openai.OpenAIModel attribute) (eval_framework.llm.vllm.BaseVLLMModel attribute) (eval_framework.llm.vllm.Qwen3_0_6B_VLLM attribute) (eval_framework.llm.vllm.Qwen3_0_6B_VLLM_No_Thinking attribute) DEG (eval_framework.tasks.base.Language attribute) DEH (eval_framework.tasks.base.Language attribute) DEI (eval_framework.tasks.base.Language attribute) DEK (eval_framework.tasks.base.Language attribute) DEL (eval_framework.tasks.base.Language attribute) DELETE (eval_framework.tasks.perturbation.PerturbationType attribute) delete_chars_in_string() (eval_framework.tasks.utils.HatPaperEditor method) delete_output_dir_after_upload (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) DEM (eval_framework.tasks.base.Language attribute) DEN (eval_framework.tasks.base.Language attribute) DEP (eval_framework.tasks.base.Language attribute) DEQ (eval_framework.tasks.base.Language attribute) DER (eval_framework.tasks.base.Language attribute) DES (eval_framework.tasks.base.Language attribute) description (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) DeterminedContext (class in eval_framework.context.determined) DEU (eval_framework.tasks.base.Language attribute) DEV (eval_framework.tasks.base.Language attribute) DEZ (eval_framework.tasks.base.Language attribute) DGA (eval_framework.tasks.base.Language attribute) DGB (eval_framework.tasks.base.Language attribute) DGC (eval_framework.tasks.base.Language attribute) DGD (eval_framework.tasks.base.Language attribute) DGE (eval_framework.tasks.base.Language attribute) DGG (eval_framework.tasks.base.Language attribute) DGH (eval_framework.tasks.base.Language attribute) DGI (eval_framework.tasks.base.Language attribute) DGK (eval_framework.tasks.base.Language attribute) DGL (eval_framework.tasks.base.Language attribute) DGN (eval_framework.tasks.base.Language attribute) DGO (eval_framework.tasks.base.Language attribute) DGR (eval_framework.tasks.base.Language attribute) DGS (eval_framework.tasks.base.Language attribute) DGT (eval_framework.tasks.base.Language attribute) DGU (eval_framework.tasks.base.Language attribute) DGW (eval_framework.tasks.base.Language attribute) DGX (eval_framework.tasks.base.Language attribute) DGZ (eval_framework.tasks.base.Language attribute) DHA (eval_framework.tasks.base.Language attribute) DHD (eval_framework.tasks.base.Language attribute) DHG (eval_framework.tasks.base.Language attribute) DHI (eval_framework.tasks.base.Language attribute) DHL (eval_framework.tasks.base.Language attribute) DHM (eval_framework.tasks.base.Language attribute) DHN (eval_framework.tasks.base.Language attribute) DHO (eval_framework.tasks.base.Language attribute) DHR (eval_framework.tasks.base.Language attribute) DHS (eval_framework.tasks.base.Language attribute) DHU (eval_framework.tasks.base.Language attribute) DHV (eval_framework.tasks.base.Language attribute) DHW (eval_framework.tasks.base.Language attribute) DHX (eval_framework.tasks.base.Language attribute) DIA (eval_framework.tasks.base.Language attribute) dialect (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSqlMetricContext attribute) DIB (eval_framework.tasks.base.Language attribute) DIC (eval_framework.tasks.base.Language attribute) DID (eval_framework.tasks.base.Language attribute) DIF (eval_framework.tasks.base.Language attribute) DIG (eval_framework.tasks.base.Language attribute) DIH (eval_framework.tasks.base.Language attribute) DII (eval_framework.tasks.base.Language attribute) DIJ (eval_framework.tasks.base.Language attribute) DIK (eval_framework.tasks.base.Language attribute) DIL (eval_framework.tasks.base.Language attribute) DIM (eval_framework.tasks.base.Language attribute) DIN (eval_framework.tasks.base.Language attribute) DIO (eval_framework.tasks.base.Language attribute) DIP (eval_framework.tasks.base.Language attribute) DIQ (eval_framework.tasks.base.Language attribute) DIR (eval_framework.tasks.base.Language attribute) DIS (eval_framework.tasks.base.Language attribute) DistributionalCorrectnessScore (class in eval_framework.metrics.loglikelihood.dcs) DIT (eval_framework.tasks.base.Language attribute) DIU (eval_framework.tasks.base.Language attribute) DIV (eval_framework.tasks.base.Language attribute) DIW (eval_framework.tasks.base.Language attribute) DIX (eval_framework.tasks.base.Language attribute) DIY (eval_framework.tasks.base.Language attribute) DIZ (eval_framework.tasks.base.Language attribute) DJA (eval_framework.tasks.base.Language attribute) DJB (eval_framework.tasks.base.Language attribute) DJC (eval_framework.tasks.base.Language attribute) DJD (eval_framework.tasks.base.Language attribute) DJE (eval_framework.tasks.base.Language attribute) DJF (eval_framework.tasks.base.Language attribute) DJI (eval_framework.tasks.base.Language attribute) DJJ (eval_framework.tasks.base.Language attribute) DJK (eval_framework.tasks.base.Language attribute) DJL (eval_framework.tasks.base.Language attribute) DJM (eval_framework.tasks.base.Language attribute) DJN (eval_framework.tasks.base.Language attribute) DJO (eval_framework.tasks.base.Language attribute) DJR (eval_framework.tasks.base.Language attribute) DJU (eval_framework.tasks.base.Language attribute) DJW (eval_framework.tasks.base.Language attribute) DKA (eval_framework.tasks.base.Language attribute) DKG (eval_framework.tasks.base.Language attribute) DKK (eval_framework.tasks.base.Language attribute) DKL (eval_framework.tasks.base.Language attribute) DKR (eval_framework.tasks.base.Language attribute) DKS (eval_framework.tasks.base.Language attribute) DKX (eval_framework.tasks.base.Language attribute) DLG (eval_framework.tasks.base.Language attribute) DLK (eval_framework.tasks.base.Language attribute) DLM (eval_framework.tasks.base.Language attribute) DLN (eval_framework.tasks.base.Language attribute) DMA (eval_framework.tasks.base.Language attribute) DMB (eval_framework.tasks.base.Language attribute) DMC (eval_framework.tasks.base.Language attribute) DMD (eval_framework.tasks.base.Language attribute) DME (eval_framework.tasks.base.Language attribute) DMF (eval_framework.tasks.base.Language attribute) DMG (eval_framework.tasks.base.Language attribute) DMK (eval_framework.tasks.base.Language attribute) DML (eval_framework.tasks.base.Language attribute) DMM (eval_framework.tasks.base.Language attribute) DMO (eval_framework.tasks.base.Language attribute) DMR (eval_framework.tasks.base.Language attribute) DMS (eval_framework.tasks.base.Language attribute) DMU (eval_framework.tasks.base.Language attribute) DMV (eval_framework.tasks.base.Language attribute) DMW (eval_framework.tasks.base.Language attribute) DMX (eval_framework.tasks.base.Language attribute) DMY (eval_framework.tasks.base.Language attribute) DNA (eval_framework.tasks.base.Language attribute) DND (eval_framework.tasks.base.Language attribute) DNE (eval_framework.tasks.base.Language attribute) DNG (eval_framework.tasks.base.Language attribute) DNI (eval_framework.tasks.base.Language attribute) DNJ (eval_framework.tasks.base.Language attribute) DNK (eval_framework.tasks.base.Language attribute) DNN (eval_framework.tasks.base.Language attribute) DNO (eval_framework.tasks.base.Language attribute) DNR (eval_framework.tasks.base.Language attribute) DNT (eval_framework.tasks.base.Language attribute) DNU (eval_framework.tasks.base.Language attribute) DNV (eval_framework.tasks.base.Language attribute) DNW (eval_framework.tasks.base.Language attribute) DNY (eval_framework.tasks.base.Language attribute) DOA (eval_framework.tasks.base.Language attribute) DOB (eval_framework.tasks.base.Language attribute) DOC (eval_framework.tasks.base.Language attribute) DOE (eval_framework.tasks.base.Language attribute) DOF (eval_framework.tasks.base.Language attribute) DOH (eval_framework.tasks.base.Language attribute) DOI (eval_framework.tasks.base.Language attribute) DOK (eval_framework.tasks.base.Language attribute) DOL (eval_framework.tasks.base.Language attribute) DON (eval_framework.tasks.base.Language attribute) DOO (eval_framework.tasks.base.Language attribute) DOP (eval_framework.tasks.base.Language attribute) DOQ (eval_framework.tasks.base.Language attribute) DOR (eval_framework.tasks.base.Language attribute) DOS (eval_framework.tasks.base.Language attribute) DOT (eval_framework.tasks.base.Language attribute) DOV (eval_framework.tasks.base.Language attribute) DOW (eval_framework.tasks.base.Language attribute) DOX (eval_framework.tasks.base.Language attribute) DOY (eval_framework.tasks.base.Language attribute) DOZ (eval_framework.tasks.base.Language attribute) DPP (eval_framework.tasks.base.Language attribute) DRB (eval_framework.tasks.base.Language attribute) DRC (eval_framework.tasks.base.Language attribute) DRD (eval_framework.tasks.base.Language attribute) DRE (eval_framework.tasks.base.Language attribute) DRG (eval_framework.tasks.base.Language attribute) DRH (eval_framework.tasks.base.Language attribute) DRI (eval_framework.tasks.base.Language attribute) DRL (eval_framework.tasks.base.Language attribute) DRN (eval_framework.tasks.base.Language attribute) DRO (eval_framework.tasks.base.Language attribute) DRQ (eval_framework.tasks.base.Language attribute) DRR (eval_framework.tasks.base.Language attribute) DRS (eval_framework.tasks.base.Language attribute) DRT (eval_framework.tasks.base.Language attribute) DRU (eval_framework.tasks.base.Language attribute) DRW (eval_framework.tasks.base.Language attribute) DRY (eval_framework.tasks.base.Language attribute) DSB (eval_framework.tasks.base.Language attribute) DSE (eval_framework.tasks.base.Language attribute) DSH (eval_framework.tasks.base.Language attribute) DSI (eval_framework.tasks.base.Language attribute) DSK (eval_framework.tasks.base.Language attribute) DSL (eval_framework.tasks.base.Language attribute) DSN (eval_framework.tasks.base.Language attribute) DSO (eval_framework.tasks.base.Language attribute) DSQ (eval_framework.tasks.base.Language attribute) DSZ (eval_framework.tasks.base.Language attribute) DTA (eval_framework.tasks.base.Language attribute) DTB (eval_framework.tasks.base.Language attribute) DTD (eval_framework.tasks.base.Language attribute) DTH (eval_framework.tasks.base.Language attribute) DTI (eval_framework.tasks.base.Language attribute) DTK (eval_framework.tasks.base.Language attribute) DTM (eval_framework.tasks.base.Language attribute) DTN (eval_framework.tasks.base.Language attribute) DTO (eval_framework.tasks.base.Language attribute) DTP (eval_framework.tasks.base.Language attribute) DTR (eval_framework.tasks.base.Language attribute) DTS (eval_framework.tasks.base.Language attribute) DTT (eval_framework.tasks.base.Language attribute) DTU (eval_framework.tasks.base.Language attribute) DTY (eval_framework.tasks.base.Language attribute) DUA (eval_framework.tasks.base.Language attribute) DUB (eval_framework.tasks.base.Language attribute) DUC (class in eval_framework.tasks.benchmarks.duc) (eval_framework.tasks.base.Language attribute) DUC_ABSTRACTIVE (class in eval_framework.tasks.benchmarks.duc) DUC_EXTRACTIVE (class in eval_framework.tasks.benchmarks.duc) DUD (eval_framework.tasks.base.Language attribute) DUE (eval_framework.tasks.base.Language attribute) DUF (eval_framework.tasks.base.Language attribute) DUG (eval_framework.tasks.base.Language attribute) DUH (eval_framework.tasks.base.Language attribute) DUI (eval_framework.tasks.base.Language attribute) DUJ (eval_framework.tasks.base.Language attribute) DUK (eval_framework.tasks.base.Language attribute) DUL (eval_framework.tasks.base.Language attribute) DUM (eval_framework.tasks.base.Language attribute) DUN (eval_framework.tasks.base.Language attribute) DUO (eval_framework.tasks.base.Language attribute) DUP (eval_framework.tasks.base.Language attribute) DUQ (eval_framework.tasks.base.Language attribute) DUR (eval_framework.tasks.base.Language attribute) DUS (eval_framework.tasks.base.Language attribute) DUU (eval_framework.tasks.base.Language attribute) DUV (eval_framework.tasks.base.Language attribute) DUW (eval_framework.tasks.base.Language attribute) DUX (eval_framework.tasks.base.Language attribute) DUY (eval_framework.tasks.base.Language attribute) DUZ (eval_framework.tasks.base.Language attribute) DVA (eval_framework.tasks.base.Language attribute) DWA (eval_framework.tasks.base.Language attribute) DWK (eval_framework.tasks.base.Language attribute) DWL (eval_framework.tasks.base.Language attribute) DWR (eval_framework.tasks.base.Language attribute) DWS (eval_framework.tasks.base.Language attribute) DWU (eval_framework.tasks.base.Language attribute) DWW (eval_framework.tasks.base.Language attribute) DWY (eval_framework.tasks.base.Language attribute) DWZ (eval_framework.tasks.base.Language attribute) DYA (eval_framework.tasks.base.Language attribute) DYB (eval_framework.tasks.base.Language attribute) DYD (eval_framework.tasks.base.Language attribute) DYG (eval_framework.tasks.base.Language attribute) DYI (eval_framework.tasks.base.Language attribute) DYK (eval_framework.tasks.base.Language attribute) DYM (eval_framework.tasks.base.Language attribute) DYN (eval_framework.tasks.base.Language attribute) DYO (eval_framework.tasks.base.Language attribute) DYR (eval_framework.tasks.base.Language attribute) DYU (eval_framework.tasks.base.Language attribute) DYY (eval_framework.tasks.base.Language attribute) DZA (eval_framework.tasks.base.Language attribute) DZD (eval_framework.tasks.base.Language attribute) DZE (eval_framework.tasks.base.Language attribute) DZG (eval_framework.tasks.base.Language attribute) DZL (eval_framework.tasks.base.Language attribute) DZN (eval_framework.tasks.base.Language attribute) DZO (eval_framework.tasks.base.Language attribute) E EAA (eval_framework.tasks.base.Language attribute) EBC (eval_framework.tasks.base.Language attribute) EBG (eval_framework.tasks.base.Language attribute) EBK (eval_framework.tasks.base.Language attribute) EBO (eval_framework.tasks.base.Language attribute) EBR (eval_framework.tasks.base.Language attribute) EBU (eval_framework.tasks.base.Language attribute) ECR (eval_framework.tasks.base.Language attribute) ECS (eval_framework.tasks.base.Language attribute) ECY (eval_framework.tasks.base.Language attribute) Editor (class in eval_framework.tasks.utils) EDITOR (eval_framework.tasks.perturbation.PerturbationType attribute) EEE (eval_framework.tasks.base.Language attribute) EFA (eval_framework.tasks.base.Language attribute) EFE (eval_framework.tasks.base.Language attribute) EFI (eval_framework.tasks.base.Language attribute) EGA (eval_framework.tasks.base.Language attribute) EGL (eval_framework.tasks.base.Language attribute) EGM (eval_framework.tasks.base.Language attribute) EGO (eval_framework.tasks.base.Language attribute) EGY (eval_framework.tasks.base.Language attribute) EHS (eval_framework.tasks.base.Language attribute) EHU (eval_framework.tasks.base.Language attribute) EIP (eval_framework.tasks.base.Language attribute) EIT (eval_framework.tasks.base.Language attribute) EIV (eval_framework.tasks.base.Language attribute) EJA (eval_framework.tasks.base.Language attribute) EKA (eval_framework.tasks.base.Language attribute) EKC (eval_framework.tasks.base.Language attribute) EKE (eval_framework.tasks.base.Language attribute) EKG (eval_framework.tasks.base.Language attribute) EKI (eval_framework.tasks.base.Language attribute) EKK (eval_framework.tasks.base.Language attribute) EKL (eval_framework.tasks.base.Language attribute) EKM (eval_framework.tasks.base.Language attribute) EKO (eval_framework.tasks.base.Language attribute) EKP (eval_framework.tasks.base.Language attribute) EKR (eval_framework.tasks.base.Language attribute) EKY (eval_framework.tasks.base.Language attribute) ELE (eval_framework.tasks.base.Language attribute) ELH (eval_framework.tasks.base.Language attribute) ELI (eval_framework.tasks.base.Language attribute) ELK (eval_framework.tasks.base.Language attribute) ELL (eval_framework.tasks.base.Language attribute) ELM (eval_framework.tasks.base.Language attribute) ELO (eval_framework.tasks.base.Language attribute) ELP (eval_framework.tasks.base.Language attribute) ELU (eval_framework.tasks.base.Language attribute) ELX (eval_framework.tasks.base.Language attribute) EMA (eval_framework.tasks.base.Language attribute) EMB (eval_framework.tasks.base.Language attribute) EME (eval_framework.tasks.base.Language attribute) EMG (eval_framework.tasks.base.Language attribute) EMI (eval_framework.tasks.base.Language attribute) EMK (eval_framework.tasks.base.Language attribute) EML (eval_framework.tasks.base.Language attribute) EMM (eval_framework.tasks.base.Language attribute) EMN (eval_framework.tasks.base.Language attribute) EMO (eval_framework.tasks.base.Language attribute) EMP (eval_framework.tasks.base.Language attribute) EMQ (eval_framework.tasks.base.Language attribute) EMS (eval_framework.tasks.base.Language attribute) EMU (eval_framework.tasks.base.Language attribute) EMW (eval_framework.tasks.base.Language attribute) EMX (eval_framework.tasks.base.Language attribute) EMY (eval_framework.tasks.base.Language attribute) EMZ (eval_framework.tasks.base.Language attribute) ENA (eval_framework.tasks.base.Language attribute) ENB (eval_framework.tasks.base.Language attribute) ENC (eval_framework.tasks.base.Language attribute) encode() (eval_framework.llm.vllm.HFTokenizerProtocol method) (eval_framework.tasks.utils.CallableSerializer static method) encode_formatted_struct() (eval_framework.llm.mistral.MistralAdapter method) (eval_framework.llm.vllm.VLLMTokenizer method) (eval_framework.llm.vllm.VLLMTokenizerAPI method) encode_plain_text() (eval_framework.llm.mistral.MistralAdapter method) (eval_framework.llm.vllm.VLLMTokenizer method) (eval_framework.llm.vllm.VLLMTokenizerAPI method) EncodingError END (eval_framework.tasks.base.Language attribute) ENF (eval_framework.tasks.base.Language attribute) ENG (eval_framework.tasks.base.Language attribute) ENH (eval_framework.tasks.base.Language attribute) ENI (eval_framework.tasks.base.Language attribute) ENL (eval_framework.tasks.base.Language attribute) ENM (eval_framework.tasks.base.Language attribute) ENN (eval_framework.tasks.base.Language attribute) ENO (eval_framework.tasks.base.Language attribute) ENQ (eval_framework.tasks.base.Language attribute) ENR (eval_framework.tasks.base.Language attribute) entry_point (eval_framework.tasks.benchmarks.humaneval.HumanEvalMetricContext attribute) ENU (eval_framework.tasks.base.Language attribute) ENV (eval_framework.tasks.base.Language attribute) ENW (eval_framework.tasks.base.Language attribute) ENX (eval_framework.tasks.base.Language attribute) EOT (eval_framework.tasks.base.Language attribute) EPI (eval_framework.tasks.base.Language attribute) EPO (eval_framework.tasks.base.Language attribute) ERA (eval_framework.tasks.base.Language attribute) ERG (eval_framework.tasks.base.Language attribute) ERH (eval_framework.tasks.base.Language attribute) ERI (eval_framework.tasks.base.Language attribute) ERK (eval_framework.tasks.base.Language attribute) ERO (eval_framework.tasks.base.Language attribute) ERR (eval_framework.tasks.base.Language attribute) error (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) ERS (eval_framework.tasks.base.Language attribute) ERT (eval_framework.tasks.base.Language attribute) ERW (eval_framework.tasks.base.Language attribute) ESE (eval_framework.tasks.base.Language attribute) ESG (eval_framework.tasks.base.Language attribute) ESH (eval_framework.tasks.base.Language attribute) ESI (eval_framework.tasks.base.Language attribute) ESK (eval_framework.tasks.base.Language attribute) ESL (eval_framework.tasks.base.Language attribute) ESM (eval_framework.tasks.base.Language attribute) ESN (eval_framework.tasks.base.Language attribute) ESO (eval_framework.tasks.base.Language attribute) ESQ (eval_framework.tasks.base.Language attribute) ESS (eval_framework.tasks.base.Language attribute) EST (eval_framework.tasks.base.Language attribute) estimate_pass_at_k() (in module eval_framework.metrics.completion.code_execution_pass_at_one) ESU (eval_framework.tasks.base.Language attribute) ESY (eval_framework.tasks.base.Language attribute) ETB (eval_framework.tasks.base.Language attribute) ETC (eval_framework.tasks.base.Language attribute) ETH (eval_framework.tasks.base.Language attribute) ETN (eval_framework.tasks.base.Language attribute) ETO (eval_framework.tasks.base.Language attribute) ETR (eval_framework.tasks.base.Language attribute) ETS (eval_framework.tasks.base.Language attribute) ETT (eval_framework.tasks.base.Language attribute) ETU (eval_framework.tasks.base.Language attribute) ETX (eval_framework.tasks.base.Language attribute) ETZ (eval_framework.tasks.base.Language attribute) EUD (eval_framework.tasks.base.Language attribute) EUR (eval_framework.tasks.base.Language attribute) EUS (eval_framework.tasks.base.Language attribute) eval_framework module eval_framework.base_config module eval_framework.context module eval_framework.context.determined module eval_framework.context.eval module eval_framework.context.local module eval_framework.evaluation_generator module eval_framework.exceptions module eval_framework.llm module eval_framework.llm.aleph_alpha module eval_framework.llm.base module eval_framework.llm.huggingface module eval_framework.llm.mistral module eval_framework.llm.models module eval_framework.llm.openai module eval_framework.llm.vllm module eval_framework.logger module eval_framework.main module eval_framework.metrics module eval_framework.metrics.base module eval_framework.metrics.completion module eval_framework.metrics.completion.accuracy_completion module eval_framework.metrics.completion.aidanbench module eval_framework.metrics.completion.bleu module eval_framework.metrics.completion.chrf module eval_framework.metrics.completion.code_assertion module eval_framework.metrics.completion.code_execution_pass_at_one module eval_framework.metrics.completion.comet module eval_framework.metrics.completion.concordance_index module eval_framework.metrics.completion.csv_format module eval_framework.metrics.completion.cwe_accuracy module eval_framework.metrics.completion.exponential_similarity module eval_framework.metrics.completion.f1 module eval_framework.metrics.completion.format_checker module eval_framework.metrics.completion.grid_difference module eval_framework.metrics.completion.ifeval module eval_framework.metrics.completion.json_format module eval_framework.metrics.completion.language_checker module eval_framework.metrics.completion.length_control module eval_framework.metrics.completion.math_reasoning_completion module eval_framework.metrics.completion.niah_accuracy module eval_framework.metrics.completion.placeholder_checker module eval_framework.metrics.completion.repetition module eval_framework.metrics.completion.rouge_1 module eval_framework.metrics.completion.rouge_2 module eval_framework.metrics.completion.rouge_geometric_mean module eval_framework.metrics.completion.rouge_l module eval_framework.metrics.completion.struct_eval_metrics module eval_framework.metrics.completion.ter module eval_framework.metrics.completion.text_counter module eval_framework.metrics.efficiency module eval_framework.metrics.efficiency.bytes_per_sequence_position module eval_framework.metrics.llm module eval_framework.metrics.llm.base module eval_framework.metrics.llm.llm_judge_chatbot_style module eval_framework.metrics.llm.llm_judge_coherence module eval_framework.metrics.llm.llm_judge_completion_accuracy module eval_framework.metrics.llm.llm_judge_conciseness module eval_framework.metrics.llm.llm_judge_contains_names module eval_framework.metrics.llm.llm_judge_format_correctness module eval_framework.metrics.llm.llm_judge_instruction module eval_framework.metrics.llm.llm_judge_mtbench_pair module eval_framework.metrics.llm.llm_judge_mtbench_single module eval_framework.metrics.llm.llm_judge_refusal module eval_framework.metrics.llm.llm_judge_sql module eval_framework.metrics.llm.llm_judge_world_knowledge module eval_framework.metrics.llm.utils module eval_framework.metrics.loglikelihood module eval_framework.metrics.loglikelihood.accuracy_loglikelihood module eval_framework.metrics.loglikelihood.base module eval_framework.metrics.loglikelihood.confidence_weighted_accuracy module eval_framework.metrics.loglikelihood.dcs module eval_framework.metrics.loglikelihood.probability_mass module eval_framework.metrics.loglikelihood.ternary module eval_framework.response_generator module eval_framework.result_processors module eval_framework.result_processors.base module eval_framework.result_processors.hf_uploader module eval_framework.result_processors.result_processor module eval_framework.result_processors.wandb_uploader module eval_framework.run module eval_framework.run_direct module eval_framework.tasks module eval_framework.tasks.base module eval_framework.tasks.benchmarks module eval_framework.tasks.benchmarks.aidanbench module eval_framework.tasks.benchmarks.arc module eval_framework.tasks.benchmarks.arc_de module eval_framework.tasks.benchmarks.arc_fi module eval_framework.tasks.benchmarks.belebele module eval_framework.tasks.benchmarks.bigcodebench module eval_framework.tasks.benchmarks.casehold module eval_framework.tasks.benchmarks.chembench module eval_framework.tasks.benchmarks.copa module eval_framework.tasks.benchmarks.duc module eval_framework.tasks.benchmarks.flores200 module eval_framework.tasks.benchmarks.flores_plus module eval_framework.tasks.benchmarks.gpqa module eval_framework.tasks.benchmarks.gsm8k module eval_framework.tasks.benchmarks.hellaswag module eval_framework.tasks.benchmarks.hellaswag_de module eval_framework.tasks.benchmarks.humaneval module eval_framework.tasks.benchmarks.ifeval module eval_framework.tasks.benchmarks.include module eval_framework.tasks.benchmarks.infinitebench module eval_framework.tasks.benchmarks.math_reasoning module eval_framework.tasks.benchmarks.mbpp module eval_framework.tasks.benchmarks.mmlu module eval_framework.tasks.benchmarks.mmlu_de module eval_framework.tasks.benchmarks.mmlu_pro module eval_framework.tasks.benchmarks.mmmlu module eval_framework.tasks.benchmarks.openbookqa module eval_framework.tasks.benchmarks.opengptx_eu20 module eval_framework.tasks.benchmarks.pawsx module eval_framework.tasks.benchmarks.piqa module eval_framework.tasks.benchmarks.quality module eval_framework.tasks.benchmarks.sciq module eval_framework.tasks.benchmarks.sphyr module eval_framework.tasks.benchmarks.squad module eval_framework.tasks.benchmarks.struct_eval module eval_framework.tasks.benchmarks.tablebench module eval_framework.tasks.benchmarks.triviaqa module eval_framework.tasks.benchmarks.truthfulqa module eval_framework.tasks.benchmarks.winogender module eval_framework.tasks.benchmarks.winogrande module eval_framework.tasks.benchmarks.winox module eval_framework.tasks.benchmarks.wmt module eval_framework.tasks.benchmarks.zero_scrolls module eval_framework.tasks.eval_config module eval_framework.tasks.perturbation module eval_framework.tasks.registry module eval_framework.tasks.task_loader module eval_framework.tasks.task_names module eval_framework.tasks.utils module EvalConfig (class in eval_framework.tasks.eval_config) EvalContext (class in eval_framework.context.eval) evaluate_csv_format() (in module eval_framework.metrics.completion.csv_format) EvaluationGenerator (class in eval_framework.evaluation_generator) EVE (eval_framework.tasks.base.Language attribute) EVH (eval_framework.tasks.base.Language attribute) EVN (eval_framework.tasks.base.Language attribute) EWE (eval_framework.tasks.base.Language attribute) EWO (eval_framework.tasks.base.Language attribute) exact_match (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) execute_python_code_with_tests() (in module eval_framework.tasks.utils) ExecutionResult (class in eval_framework.tasks.utils) ExponentialSimilarity (class in eval_framework.metrics.completion.exponential_similarity) EXT (eval_framework.tasks.base.Language attribute) extra_task_modules (eval_framework.context.determined.Hyperparameters attribute) extract_csv_from_text() (in module eval_framework.metrics.completion.csv_format) extract_executable_code() (in module eval_framework.tasks.benchmarks.bigcodebench) extract_grid_from_prompt() (eval_framework.metrics.completion.grid_difference.GridDifference method) extract_imports() (in module eval_framework.tasks.utils) extract_last_two_dollar_text() (eval_framework.tasks.benchmarks.math_reasoning.MATH method) extract_query_from_completions() (in module eval_framework.metrics.llm.llm_judge_sql) EYA (eval_framework.tasks.base.Language attribute) EYO (eval_framework.tasks.base.Language attribute) EZA (eval_framework.tasks.base.Language attribute) EZE (eval_framework.tasks.base.Language attribute) F F1 (class in eval_framework.metrics.completion.f1) FAA (eval_framework.tasks.base.Language attribute) FAB (eval_framework.tasks.base.Language attribute) FAD (eval_framework.tasks.base.Language attribute) FAF (eval_framework.tasks.base.Language attribute) FAG (eval_framework.tasks.base.Language attribute) FAH (eval_framework.tasks.base.Language attribute) FAI (eval_framework.tasks.base.Language attribute) FAJ (eval_framework.tasks.base.Language attribute) FAK (eval_framework.tasks.base.Language attribute) FAL (eval_framework.tasks.base.Language attribute) FAM (eval_framework.tasks.base.Language attribute) FAN (eval_framework.tasks.base.Language attribute) FAO (eval_framework.tasks.base.Language attribute) FAP (eval_framework.tasks.base.Language attribute) FAR (eval_framework.tasks.base.Language attribute) FAS (eval_framework.tasks.base.Language attribute) FAT (eval_framework.tasks.base.Language attribute) FAU (eval_framework.tasks.base.Language attribute) FAX (eval_framework.tasks.base.Language attribute) FAY (eval_framework.tasks.base.Language attribute) FAZ (eval_framework.tasks.base.Language attribute) FBL (eval_framework.tasks.base.Language attribute) FCS (eval_framework.tasks.base.Language attribute) FER (eval_framework.tasks.base.Language attribute) FEWSHOT_ITEMS (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) FEWSHOT_SPLIT (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8K attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeDebug attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeRun attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnDia attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnMC attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnQA attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_MathFind attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveKV2 attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveNumber attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrievePassKey1 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2025 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_FR attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winox.WINOX attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_COMPLETION attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) FFI (eval_framework.tasks.base.Language attribute) FFM (eval_framework.tasks.base.Language attribute) FGR (eval_framework.tasks.base.Language attribute) FIA (eval_framework.tasks.base.Language attribute) FIE (eval_framework.tasks.base.Language attribute) FIF (eval_framework.tasks.base.Language attribute) FIJ (eval_framework.tasks.base.Language attribute) FIL (eval_framework.tasks.base.Language attribute) FIN (eval_framework.tasks.base.Language attribute) find_all_python_files() (in module eval_framework.tasks.task_loader) FIP (eval_framework.tasks.base.Language attribute) FIR (eval_framework.tasks.base.Language attribute) FIT (eval_framework.tasks.base.Language attribute) FIW (eval_framework.tasks.base.Language attribute) FIZ (eval_framework.tasks.base.Language attribute) FKK (eval_framework.tasks.base.Language attribute) FKV (eval_framework.tasks.base.Language attribute) FLA (eval_framework.tasks.base.Language attribute) FLH (eval_framework.tasks.base.Language attribute) FLI (eval_framework.tasks.base.Language attribute) FLL (eval_framework.tasks.base.Language attribute) FLM (eval_framework.tasks.base.Language attribute) FLN (eval_framework.tasks.base.Language attribute) Flores200 (class in eval_framework.tasks.benchmarks.flores200) FloresPlus (class in eval_framework.tasks.benchmarks.flores_plus) FLR (eval_framework.tasks.base.Language attribute) FLY (eval_framework.tasks.base.Language attribute) FMP (eval_framework.tasks.base.Language attribute) FMU (eval_framework.tasks.base.Language attribute) FNB (eval_framework.tasks.base.Language attribute) FNG (eval_framework.tasks.base.Language attribute) FNI (eval_framework.tasks.base.Language attribute) FOD (eval_framework.tasks.base.Language attribute) FOI (eval_framework.tasks.base.Language attribute) FOM (eval_framework.tasks.base.Language attribute) FON (eval_framework.tasks.base.Language attribute) FOR (eval_framework.tasks.base.Language attribute) formatter_output_mode (eval_framework.llm.mistral.MistralVLLM property) (eval_framework.llm.vllm.BaseVLLMModel property) FOS (eval_framework.tasks.base.Language attribute) FPE (eval_framework.tasks.base.Language attribute) FQS (eval_framework.tasks.base.Language attribute) FRA (eval_framework.tasks.base.Language attribute) FRC (eval_framework.tasks.base.Language attribute) FRD (eval_framework.tasks.base.Language attribute) FRI (eval_framework.tasks.base.Language attribute) FRK (eval_framework.tasks.base.Language attribute) FRM (eval_framework.tasks.base.Language attribute) FRO (eval_framework.tasks.base.Language attribute) from_context() (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext class method) from_yaml() (eval_framework.base_config.BaseConfig class method) FRP (eval_framework.tasks.base.Language attribute) FRQ (eval_framework.tasks.base.Language attribute) FRR (eval_framework.tasks.base.Language attribute) FRS (eval_framework.tasks.base.Language attribute) FRT (eval_framework.tasks.base.Language attribute) FRY (eval_framework.tasks.base.Language attribute) FSE (eval_framework.tasks.base.Language attribute) FSL (eval_framework.tasks.base.Language attribute) FSS (eval_framework.tasks.base.Language attribute) FUB (eval_framework.tasks.base.Language attribute) FUC (eval_framework.tasks.base.Language attribute) FUD (eval_framework.tasks.base.Language attribute) FUE (eval_framework.tasks.base.Language attribute) FUF (eval_framework.tasks.base.Language attribute) FUH (eval_framework.tasks.base.Language attribute) FUI (eval_framework.tasks.base.Language attribute) FUJ (eval_framework.tasks.base.Language attribute) FUL (eval_framework.tasks.base.Language attribute) fulfills_schema (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) FullTextMMLU (class in eval_framework.tasks.benchmarks.mmlu) FUM (eval_framework.tasks.base.Language attribute) FUN (eval_framework.tasks.base.Language attribute) FUQ (eval_framework.tasks.base.Language attribute) FUR (eval_framework.tasks.base.Language attribute) FUT (eval_framework.tasks.base.Language attribute) FUU (eval_framework.tasks.base.Language attribute) FUV (eval_framework.tasks.base.Language attribute) FUY (eval_framework.tasks.base.Language attribute) FVR (eval_framework.tasks.base.Language attribute) FWA (eval_framework.tasks.base.Language attribute) FWE (eval_framework.tasks.base.Language attribute) G GAA (eval_framework.tasks.base.Language attribute) GAB (eval_framework.tasks.base.Language attribute) GAC (eval_framework.tasks.base.Language attribute) GAD (eval_framework.tasks.base.Language attribute) GAE (eval_framework.tasks.base.Language attribute) GAF (eval_framework.tasks.base.Language attribute) GAG (eval_framework.tasks.base.Language attribute) GAH (eval_framework.tasks.base.Language attribute) GAI (eval_framework.tasks.base.Language attribute) GAJ (eval_framework.tasks.base.Language attribute) GAK (eval_framework.tasks.base.Language attribute) GAL (eval_framework.tasks.base.Language attribute) GAM (eval_framework.tasks.base.Language attribute) GAN (eval_framework.tasks.base.Language attribute) GAO (eval_framework.tasks.base.Language attribute) GAP (eval_framework.tasks.base.Language attribute) GAQ (eval_framework.tasks.base.Language attribute) GAR (eval_framework.tasks.base.Language attribute) GAS (eval_framework.tasks.base.Language attribute) GAT (eval_framework.tasks.base.Language attribute) GAU (eval_framework.tasks.base.Language attribute) GAV (eval_framework.tasks.base.Language attribute) GAW (eval_framework.tasks.base.Language attribute) GAX (eval_framework.tasks.base.Language attribute) GAY (eval_framework.tasks.base.Language attribute) GAZ (eval_framework.tasks.base.Language attribute) GBA (eval_framework.tasks.base.Language attribute) GBB (eval_framework.tasks.base.Language attribute) GBC (eval_framework.tasks.base.Language attribute) GBD (eval_framework.tasks.base.Language attribute) GBE (eval_framework.tasks.base.Language attribute) GBF (eval_framework.tasks.base.Language attribute) GBG (eval_framework.tasks.base.Language attribute) GBH (eval_framework.tasks.base.Language attribute) GBI (eval_framework.tasks.base.Language attribute) GBJ (eval_framework.tasks.base.Language attribute) GBK (eval_framework.tasks.base.Language attribute) GBL (eval_framework.tasks.base.Language attribute) GBM (eval_framework.tasks.base.Language attribute) GBN (eval_framework.tasks.base.Language attribute) GBO (eval_framework.tasks.base.Language attribute) GBP (eval_framework.tasks.base.Language attribute) GBQ (eval_framework.tasks.base.Language attribute) GBR (eval_framework.tasks.base.Language attribute) GBS (eval_framework.tasks.base.Language attribute) GBU (eval_framework.tasks.base.Language attribute) GBV (eval_framework.tasks.base.Language attribute) GBW (eval_framework.tasks.base.Language attribute) GBX (eval_framework.tasks.base.Language attribute) GBY (eval_framework.tasks.base.Language attribute) GBZ (eval_framework.tasks.base.Language attribute) GCC (eval_framework.tasks.base.Language attribute) GCD (eval_framework.tasks.base.Language attribute) GCE (eval_framework.tasks.base.Language attribute) GCF (eval_framework.tasks.base.Language attribute) GCL (eval_framework.tasks.base.Language attribute) GCN (eval_framework.tasks.base.Language attribute) GCR (eval_framework.tasks.base.Language attribute) GCT (eval_framework.tasks.base.Language attribute) GDA (eval_framework.tasks.base.Language attribute) GDB (eval_framework.tasks.base.Language attribute) GDC (eval_framework.tasks.base.Language attribute) GDD (eval_framework.tasks.base.Language attribute) GDE (eval_framework.tasks.base.Language attribute) GDF (eval_framework.tasks.base.Language attribute) GDG (eval_framework.tasks.base.Language attribute) GDH (eval_framework.tasks.base.Language attribute) GDI (eval_framework.tasks.base.Language attribute) GDJ (eval_framework.tasks.base.Language attribute) GDK (eval_framework.tasks.base.Language attribute) GDL (eval_framework.tasks.base.Language attribute) GDM (eval_framework.tasks.base.Language attribute) GDN (eval_framework.tasks.base.Language attribute) GDO (eval_framework.tasks.base.Language attribute) GDQ (eval_framework.tasks.base.Language attribute) GDR (eval_framework.tasks.base.Language attribute) GDS (eval_framework.tasks.base.Language attribute) GDT (eval_framework.tasks.base.Language attribute) GDU (eval_framework.tasks.base.Language attribute) GDX (eval_framework.tasks.base.Language attribute) GEA (eval_framework.tasks.base.Language attribute) GEB (eval_framework.tasks.base.Language attribute) GEC (eval_framework.tasks.base.Language attribute) GED (eval_framework.tasks.base.Language attribute) GEF (eval_framework.tasks.base.Language attribute) GEG (eval_framework.tasks.base.Language attribute) GEH (eval_framework.tasks.base.Language attribute) GEI (eval_framework.tasks.base.Language attribute) GEJ (eval_framework.tasks.base.Language attribute) GEK (eval_framework.tasks.base.Language attribute) GEL (eval_framework.tasks.base.Language attribute) GEN (eval_framework.tasks.base.Language attribute) generate() (eval_framework.llm.base.BaseLLM method) (eval_framework.response_generator.ResponseGenerator method) generate_completions() (eval_framework.tasks.base.BaseTask method) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal method) generate_embeddings() (eval_framework.llm.openai.OpenAIEmbeddingModel method) generate_from_messages() (eval_framework.llm.aleph_alpha.AlephAlphaAPIModel method) (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.huggingface.BaseHFLLM method) (eval_framework.llm.openai.OpenAIEmbeddingModel method) (eval_framework.llm.openai.OpenAIModel method) (eval_framework.llm.vllm.BaseVLLMModel method) generate_from_samples() (eval_framework.llm.base.BaseLLM method) generate_output_dir() (in module eval_framework.result_processors.result_processor) generate_pair_judge_prompts() (in module eval_framework.metrics.llm.llm_judge_mtbench_pair) generate_single_judge_prompts() (in module eval_framework.metrics.llm.llm_judge_mtbench_single) GEQ (eval_framework.tasks.base.Language attribute) GermanCompletionChecker (class in eval_framework.metrics.completion.language_checker) GES (eval_framework.tasks.base.Language attribute) get_datasets_needing_update() (in module eval_framework.tasks.task_names) get_docker_address() (in module eval_framework.tasks.utils) get_external_dependencies() (in module eval_framework.tasks.utils) get_json_object() (in module eval_framework.metrics.completion.json_format) get_metadata() (eval_framework.tasks.base.BaseTask method) get_n_letters() (in module eval_framework.tasks.utils) get_task() (in module eval_framework.tasks.registry) get_trial_id() (eval_framework.context.determined.DeterminedContext method) (eval_framework.context.eval.EvalContext method) GEV (eval_framework.tasks.base.Language attribute) GEW (eval_framework.tasks.base.Language attribute) GEX (eval_framework.tasks.base.Language attribute) GEY (eval_framework.tasks.base.Language attribute) GEZ (eval_framework.tasks.base.Language attribute) GFK (eval_framework.tasks.base.Language attribute) GFT (eval_framework.tasks.base.Language attribute) GFX (eval_framework.tasks.base.Language attribute) GGA (eval_framework.tasks.base.Language attribute) GGB (eval_framework.tasks.base.Language attribute) GGD (eval_framework.tasks.base.Language attribute) GGE (eval_framework.tasks.base.Language attribute) GGG (eval_framework.tasks.base.Language attribute) GGH (eval_framework.tasks.base.Language attribute) GGK (eval_framework.tasks.base.Language attribute) GGL (eval_framework.tasks.base.Language attribute) GGM (eval_framework.tasks.base.Language attribute) GGN (eval_framework.tasks.base.Language attribute) GGO (eval_framework.tasks.base.Language attribute) GGR (eval_framework.tasks.base.Language attribute) GGT (eval_framework.tasks.base.Language attribute) GGU (eval_framework.tasks.base.Language attribute) GGW (eval_framework.tasks.base.Language attribute) GHA (eval_framework.tasks.base.Language attribute) GHC (eval_framework.tasks.base.Language attribute) GHE (eval_framework.tasks.base.Language attribute) GHH (eval_framework.tasks.base.Language attribute) GHK (eval_framework.tasks.base.Language attribute) GHL (eval_framework.tasks.base.Language attribute) GHN (eval_framework.tasks.base.Language attribute) GHO (eval_framework.tasks.base.Language attribute) GHR (eval_framework.tasks.base.Language attribute) GHS (eval_framework.tasks.base.Language attribute) GHT (eval_framework.tasks.base.Language attribute) GIA (eval_framework.tasks.base.Language attribute) GIB (eval_framework.tasks.base.Language attribute) GIC (eval_framework.tasks.base.Language attribute) GID (eval_framework.tasks.base.Language attribute) GIE (eval_framework.tasks.base.Language attribute) GIG (eval_framework.tasks.base.Language attribute) GIH (eval_framework.tasks.base.Language attribute) GII (eval_framework.tasks.base.Language attribute) GIL (eval_framework.tasks.base.Language attribute) GIM (eval_framework.tasks.base.Language attribute) GIN (eval_framework.tasks.base.Language attribute) GIO (eval_framework.tasks.base.Language attribute) GIP (eval_framework.tasks.base.Language attribute) GIQ (eval_framework.tasks.base.Language attribute) GIR (eval_framework.tasks.base.Language attribute) GIS (eval_framework.tasks.base.Language attribute) GIT (eval_framework.tasks.base.Language attribute) GIU (eval_framework.tasks.base.Language attribute) GIW (eval_framework.tasks.base.Language attribute) GIX (eval_framework.tasks.base.Language attribute) GIY (eval_framework.tasks.base.Language attribute) GIZ (eval_framework.tasks.base.Language attribute) GJI (eval_framework.tasks.base.Language attribute) GJK (eval_framework.tasks.base.Language attribute) GJM (eval_framework.tasks.base.Language attribute) GJN (eval_framework.tasks.base.Language attribute) GJR (eval_framework.tasks.base.Language attribute) GJU (eval_framework.tasks.base.Language attribute) GKA (eval_framework.tasks.base.Language attribute) GKD (eval_framework.tasks.base.Language attribute) GKE (eval_framework.tasks.base.Language attribute) GKN (eval_framework.tasks.base.Language attribute) GKO (eval_framework.tasks.base.Language attribute) GKP (eval_framework.tasks.base.Language attribute) GKU (eval_framework.tasks.base.Language attribute) GLA (eval_framework.tasks.base.Language attribute) GLB (eval_framework.tasks.base.Language attribute) GLC (eval_framework.tasks.base.Language attribute) GLD (eval_framework.tasks.base.Language attribute) GLE (eval_framework.tasks.base.Language attribute) GLG (eval_framework.tasks.base.Language attribute) GLH (eval_framework.tasks.base.Language attribute) GLI (eval_framework.tasks.base.Language attribute) GLJ (eval_framework.tasks.base.Language attribute) GLK (eval_framework.tasks.base.Language attribute) GLL (eval_framework.tasks.base.Language attribute) GLO (eval_framework.tasks.base.Language attribute) GLR (eval_framework.tasks.base.Language attribute) GLU (eval_framework.tasks.base.Language attribute) GLV (eval_framework.tasks.base.Language attribute) GLW (eval_framework.tasks.base.Language attribute) GLY (eval_framework.tasks.base.Language attribute) GMA (eval_framework.tasks.base.Language attribute) GMB (eval_framework.tasks.base.Language attribute) GMD (eval_framework.tasks.base.Language attribute) GMG (eval_framework.tasks.base.Language attribute) GMH (eval_framework.tasks.base.Language attribute) GML (eval_framework.tasks.base.Language attribute) GMM (eval_framework.tasks.base.Language attribute) GMN (eval_framework.tasks.base.Language attribute) GMO (eval_framework.tasks.base.Language attribute) GMR (eval_framework.tasks.base.Language attribute) GMU (eval_framework.tasks.base.Language attribute) GMV (eval_framework.tasks.base.Language attribute) GMX (eval_framework.tasks.base.Language attribute) GMY (eval_framework.tasks.base.Language attribute) GMZ (eval_framework.tasks.base.Language attribute) GNA (eval_framework.tasks.base.Language attribute) GNB (eval_framework.tasks.base.Language attribute) GNC (eval_framework.tasks.base.Language attribute) GND (eval_framework.tasks.base.Language attribute) GNE (eval_framework.tasks.base.Language attribute) GNG (eval_framework.tasks.base.Language attribute) GNH (eval_framework.tasks.base.Language attribute) GNI (eval_framework.tasks.base.Language attribute) GNJ (eval_framework.tasks.base.Language attribute) GNK (eval_framework.tasks.base.Language attribute) GNL (eval_framework.tasks.base.Language attribute) GNM (eval_framework.tasks.base.Language attribute) GNN (eval_framework.tasks.base.Language attribute) GNO (eval_framework.tasks.base.Language attribute) GNQ (eval_framework.tasks.base.Language attribute) GNR (eval_framework.tasks.base.Language attribute) GNT (eval_framework.tasks.base.Language attribute) GNU (eval_framework.tasks.base.Language attribute) GNW (eval_framework.tasks.base.Language attribute) GNZ (eval_framework.tasks.base.Language attribute) GOA (eval_framework.tasks.base.Language attribute) GOB (eval_framework.tasks.base.Language attribute) GOC (eval_framework.tasks.base.Language attribute) GOD (eval_framework.tasks.base.Language attribute) GOE (eval_framework.tasks.base.Language attribute) GOF (eval_framework.tasks.base.Language attribute) GOG (eval_framework.tasks.base.Language attribute) GOH (eval_framework.tasks.base.Language attribute) GOI (eval_framework.tasks.base.Language attribute) GOJ (eval_framework.tasks.base.Language attribute) GOK (eval_framework.tasks.base.Language attribute) GOL (eval_framework.tasks.base.Language attribute) GOM (eval_framework.tasks.base.Language attribute) GON (eval_framework.tasks.base.Language attribute) GOO (eval_framework.tasks.base.Language attribute) GOP (eval_framework.tasks.base.Language attribute) GOQ (eval_framework.tasks.base.Language attribute) GOR (eval_framework.tasks.base.Language attribute) GOS (eval_framework.tasks.base.Language attribute) GOT (eval_framework.tasks.base.Language attribute) GOU (eval_framework.tasks.base.Language attribute) GOV (eval_framework.tasks.base.Language attribute) GOW (eval_framework.tasks.base.Language attribute) GOX (eval_framework.tasks.base.Language attribute) GOY (eval_framework.tasks.base.Language attribute) GOZ (eval_framework.tasks.base.Language attribute) GPA (eval_framework.tasks.base.Language attribute) GPE (eval_framework.tasks.base.Language attribute) GPN (eval_framework.tasks.base.Language attribute) GPQA (class in eval_framework.tasks.benchmarks.gpqa) GPQA_COT (class in eval_framework.tasks.benchmarks.gpqa) GPQA_IDK (class in eval_framework.tasks.benchmarks.gpqa) GQA (eval_framework.tasks.base.Language attribute) GQI (eval_framework.tasks.base.Language attribute) GQN (eval_framework.tasks.base.Language attribute) GQR (eval_framework.tasks.base.Language attribute) GQU (eval_framework.tasks.base.Language attribute) GRA (eval_framework.tasks.base.Language attribute) GRB (eval_framework.tasks.base.Language attribute) GRC (eval_framework.tasks.base.Language attribute) GRD (eval_framework.tasks.base.Language attribute) GRG (eval_framework.tasks.base.Language attribute) GRH (eval_framework.tasks.base.Language attribute) GRI (eval_framework.tasks.base.Language attribute) GridDifference (class in eval_framework.metrics.completion.grid_difference) GRJ (eval_framework.tasks.base.Language attribute) GRM (eval_framework.tasks.base.Language attribute) GRN (eval_framework.tasks.base.Language attribute) GRO (eval_framework.tasks.base.Language attribute) ground_truth (eval_framework.tasks.base.Sample attribute) GRQ (eval_framework.tasks.base.Language attribute) GRR (eval_framework.tasks.base.Language attribute) GRS (eval_framework.tasks.base.Language attribute) GRT (eval_framework.tasks.base.Language attribute) GRU (eval_framework.tasks.base.Language attribute) GRV (eval_framework.tasks.base.Language attribute) GRW (eval_framework.tasks.base.Language attribute) GRX (eval_framework.tasks.base.Language attribute) GRY (eval_framework.tasks.base.Language attribute) GRZ (eval_framework.tasks.base.Language attribute) GSC (eval_framework.tasks.base.Language attribute) GSE (eval_framework.tasks.base.Language attribute) GSG (eval_framework.tasks.base.Language attribute) GSL (eval_framework.tasks.base.Language attribute) GSM (eval_framework.tasks.base.Language attribute) GSM8K (class in eval_framework.tasks.benchmarks.gsm8k) GSM8K_EU20_DE (class in eval_framework.tasks.benchmarks.opengptx_eu20) GSM8K_EU20_FR (class in eval_framework.tasks.benchmarks.opengptx_eu20) GSM8KEvalHarness (class in eval_framework.tasks.benchmarks.gsm8k) GSM8KReasoning (class in eval_framework.tasks.benchmarks.math_reasoning) GSN (eval_framework.tasks.base.Language attribute) GSO (eval_framework.tasks.base.Language attribute) GSP (eval_framework.tasks.base.Language attribute) GSS (eval_framework.tasks.base.Language attribute) GSW (eval_framework.tasks.base.Language attribute) GTA (eval_framework.tasks.base.Language attribute) GTI (eval_framework.tasks.base.Language attribute) GTU (eval_framework.tasks.base.Language attribute) GUA (eval_framework.tasks.base.Language attribute) GUB (eval_framework.tasks.base.Language attribute) GUC (eval_framework.tasks.base.Language attribute) GUD (eval_framework.tasks.base.Language attribute) GUE (eval_framework.tasks.base.Language attribute) GUF (eval_framework.tasks.base.Language attribute) GUG (eval_framework.tasks.base.Language attribute) GUH (eval_framework.tasks.base.Language attribute) GUI (eval_framework.tasks.base.Language attribute) GUJ (eval_framework.tasks.base.Language attribute) GUK (eval_framework.tasks.base.Language attribute) GUL (eval_framework.tasks.base.Language attribute) GUM (eval_framework.tasks.base.Language attribute) GUN (eval_framework.tasks.base.Language attribute) GUO (eval_framework.tasks.base.Language attribute) GUP (eval_framework.tasks.base.Language attribute) GUQ (eval_framework.tasks.base.Language attribute) GUR (eval_framework.tasks.base.Language attribute) GUS (eval_framework.tasks.base.Language attribute) GUT (eval_framework.tasks.base.Language attribute) GUU (eval_framework.tasks.base.Language attribute) GUV (eval_framework.tasks.base.Language attribute) GUW (eval_framework.tasks.base.Language attribute) GUX (eval_framework.tasks.base.Language attribute) GUZ (eval_framework.tasks.base.Language attribute) GVA (eval_framework.tasks.base.Language attribute) GVC (eval_framework.tasks.base.Language attribute) GVE (eval_framework.tasks.base.Language attribute) GVF (eval_framework.tasks.base.Language attribute) GVJ (eval_framework.tasks.base.Language attribute) GVL (eval_framework.tasks.base.Language attribute) GVM (eval_framework.tasks.base.Language attribute) GVN (eval_framework.tasks.base.Language attribute) GVO (eval_framework.tasks.base.Language attribute) GVP (eval_framework.tasks.base.Language attribute) GVR (eval_framework.tasks.base.Language attribute) GVS (eval_framework.tasks.base.Language attribute) GVY (eval_framework.tasks.base.Language attribute) GWA (eval_framework.tasks.base.Language attribute) GWB (eval_framework.tasks.base.Language attribute) GWC (eval_framework.tasks.base.Language attribute) GWD (eval_framework.tasks.base.Language attribute) GWE (eval_framework.tasks.base.Language attribute) GWF (eval_framework.tasks.base.Language attribute) GWG (eval_framework.tasks.base.Language attribute) GWI (eval_framework.tasks.base.Language attribute) GWJ (eval_framework.tasks.base.Language attribute) GWM (eval_framework.tasks.base.Language attribute) GWN (eval_framework.tasks.base.Language attribute) GWR (eval_framework.tasks.base.Language attribute) GWT (eval_framework.tasks.base.Language attribute) GWU (eval_framework.tasks.base.Language attribute) GWW (eval_framework.tasks.base.Language attribute) GWX (eval_framework.tasks.base.Language attribute) GXX (eval_framework.tasks.base.Language attribute) GYA (eval_framework.tasks.base.Language attribute) GYB (eval_framework.tasks.base.Language attribute) GYD (eval_framework.tasks.base.Language attribute) GYE (eval_framework.tasks.base.Language attribute) GYF (eval_framework.tasks.base.Language attribute) GYG (eval_framework.tasks.base.Language attribute) GYI (eval_framework.tasks.base.Language attribute) GYL (eval_framework.tasks.base.Language attribute) GYM (eval_framework.tasks.base.Language attribute) GYN (eval_framework.tasks.base.Language attribute) GYO (eval_framework.tasks.base.Language attribute) GYR (eval_framework.tasks.base.Language attribute) GYY (eval_framework.tasks.base.Language attribute) GYZ (eval_framework.tasks.base.Language attribute) GZA (eval_framework.tasks.base.Language attribute) GZI (eval_framework.tasks.base.Language attribute) GZN (eval_framework.tasks.base.Language attribute) H HAA (eval_framework.tasks.base.Language attribute) HAB (eval_framework.tasks.base.Language attribute) HAC (eval_framework.tasks.base.Language attribute) HAD (eval_framework.tasks.base.Language attribute) HAE (eval_framework.tasks.base.Language attribute) HAF (eval_framework.tasks.base.Language attribute) HAG (eval_framework.tasks.base.Language attribute) HAH (eval_framework.tasks.base.Language attribute) HAI (eval_framework.tasks.base.Language attribute) HAJ (eval_framework.tasks.base.Language attribute) HAK (eval_framework.tasks.base.Language attribute) HAL (eval_framework.tasks.base.Language attribute) HAM (eval_framework.tasks.base.Language attribute) HAN (eval_framework.tasks.base.Language attribute) HAO (eval_framework.tasks.base.Language attribute) HAP (eval_framework.tasks.base.Language attribute) HAQ (eval_framework.tasks.base.Language attribute) HAR (eval_framework.tasks.base.Language attribute) HAS (eval_framework.tasks.base.Language attribute) has_csv (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) HAT (eval_framework.tasks.base.Language attribute) HatPaperEditor (class in eval_framework.tasks.utils) HAU (eval_framework.tasks.base.Language attribute) HAV (eval_framework.tasks.base.Language attribute) HAW (eval_framework.tasks.base.Language attribute) HAX (eval_framework.tasks.base.Language attribute) HAY (eval_framework.tasks.base.Language attribute) HAZ (eval_framework.tasks.base.Language attribute) HBA (eval_framework.tasks.base.Language attribute) HBB (eval_framework.tasks.base.Language attribute) HBN (eval_framework.tasks.base.Language attribute) HBO (eval_framework.tasks.base.Language attribute) HBS (eval_framework.tasks.base.Language attribute) HBU (eval_framework.tasks.base.Language attribute) HCA (eval_framework.tasks.base.Language attribute) HCH (eval_framework.tasks.base.Language attribute) HDN (eval_framework.tasks.base.Language attribute) HDS (eval_framework.tasks.base.Language attribute) HDY (eval_framework.tasks.base.Language attribute) HEA (eval_framework.tasks.base.Language attribute) HEB (eval_framework.tasks.base.Language attribute) HED (eval_framework.tasks.base.Language attribute) HEG (eval_framework.tasks.base.Language attribute) HEH (eval_framework.tasks.base.Language attribute) HEI (eval_framework.tasks.base.Language attribute) HELLASWAG (class in eval_framework.tasks.benchmarks.hellaswag) HELLASWAG_DE (class in eval_framework.tasks.benchmarks.hellaswag_de) HELLASWAG_EU20_DE (class in eval_framework.tasks.benchmarks.opengptx_eu20) HELLASWAG_EU20_FR (class in eval_framework.tasks.benchmarks.opengptx_eu20) HELLASWAG_IDK (class in eval_framework.tasks.benchmarks.hellaswag) HEM (eval_framework.tasks.base.Language attribute) HER (eval_framework.tasks.base.Language attribute) hf_revision (eval_framework.context.determined.TaskArgs attribute) HF_REVISION (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) hf_revision (eval_framework.tasks.eval_config.EvalConfig attribute) hf_upload_dir (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) hf_upload_repo (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) HFLLM (class in eval_framework.llm.huggingface) HFLLM_from_name (class in eval_framework.llm.huggingface) HFLLMRegistryModel (class in eval_framework.llm.huggingface) HFTokenizerProtocol (class in eval_framework.llm.vllm) HFUploader (class in eval_framework.result_processors.hf_uploader) HGM (eval_framework.tasks.base.Language attribute) HGW (eval_framework.tasks.base.Language attribute) HHI (eval_framework.tasks.base.Language attribute) HHR (eval_framework.tasks.base.Language attribute) HHY (eval_framework.tasks.base.Language attribute) HIA (eval_framework.tasks.base.Language attribute) HIB (eval_framework.tasks.base.Language attribute) HID (eval_framework.tasks.base.Language attribute) HIF (eval_framework.tasks.base.Language attribute) HIG (eval_framework.tasks.base.Language attribute) higher_is_better (eval_framework.metrics.base.MetricResult attribute) HIGHER_IS_BETTER (eval_framework.metrics.completion.repetition.WordRepetition attribute) higher_is_better (eval_framework.result_processors.base.Result attribute) HIH (eval_framework.tasks.base.Language attribute) HII (eval_framework.tasks.base.Language attribute) HIJ (eval_framework.tasks.base.Language attribute) HIK (eval_framework.tasks.base.Language attribute) HIL (eval_framework.tasks.base.Language attribute) HIN (eval_framework.tasks.base.Language attribute) HIO (eval_framework.tasks.base.Language attribute) HIR (eval_framework.tasks.base.Language attribute) HIT (eval_framework.tasks.base.Language attribute) HIW (eval_framework.tasks.base.Language attribute) HIX (eval_framework.tasks.base.Language attribute) HJI (eval_framework.tasks.base.Language attribute) HKA (eval_framework.tasks.base.Language attribute) HKE (eval_framework.tasks.base.Language attribute) HKH (eval_framework.tasks.base.Language attribute) HKK (eval_framework.tasks.base.Language attribute) HKN (eval_framework.tasks.base.Language attribute) HKS (eval_framework.tasks.base.Language attribute) HLA (eval_framework.tasks.base.Language attribute) HLB (eval_framework.tasks.base.Language attribute) HLD (eval_framework.tasks.base.Language attribute) HLE (eval_framework.tasks.base.Language attribute) HLT (eval_framework.tasks.base.Language attribute) HLU (eval_framework.tasks.base.Language attribute) HMA (eval_framework.tasks.base.Language attribute) HMB (eval_framework.tasks.base.Language attribute) HMC (eval_framework.tasks.base.Language attribute) HMD (eval_framework.tasks.base.Language attribute) HME (eval_framework.tasks.base.Language attribute) HMF (eval_framework.tasks.base.Language attribute) HMG (eval_framework.tasks.base.Language attribute) HMH (eval_framework.tasks.base.Language attribute) HMI (eval_framework.tasks.base.Language attribute) HMJ (eval_framework.tasks.base.Language attribute) HMK (eval_framework.tasks.base.Language attribute) HML (eval_framework.tasks.base.Language attribute) HMM (eval_framework.tasks.base.Language attribute) HMN (eval_framework.tasks.base.Language attribute) HMO (eval_framework.tasks.base.Language attribute) HMP (eval_framework.tasks.base.Language attribute) HMQ (eval_framework.tasks.base.Language attribute) HMR (eval_framework.tasks.base.Language attribute) HMS (eval_framework.tasks.base.Language attribute) HMT (eval_framework.tasks.base.Language attribute) HMU (eval_framework.tasks.base.Language attribute) HMV (eval_framework.tasks.base.Language attribute) HMW (eval_framework.tasks.base.Language attribute) HMY (eval_framework.tasks.base.Language attribute) HMZ (eval_framework.tasks.base.Language attribute) HNA (eval_framework.tasks.base.Language attribute) HND (eval_framework.tasks.base.Language attribute) HNE (eval_framework.tasks.base.Language attribute) HNG (eval_framework.tasks.base.Language attribute) HNH (eval_framework.tasks.base.Language attribute) HNI (eval_framework.tasks.base.Language attribute) HNJ (eval_framework.tasks.base.Language attribute) HNM (eval_framework.tasks.base.Language attribute) HNN (eval_framework.tasks.base.Language attribute) HNO (eval_framework.tasks.base.Language attribute) HNS (eval_framework.tasks.base.Language attribute) HNU (eval_framework.tasks.base.Language attribute) HOA (eval_framework.tasks.base.Language attribute) HOB (eval_framework.tasks.base.Language attribute) HOC (eval_framework.tasks.base.Language attribute) HOD (eval_framework.tasks.base.Language attribute) HOE (eval_framework.tasks.base.Language attribute) HOH (eval_framework.tasks.base.Language attribute) HOI (eval_framework.tasks.base.Language attribute) HOJ (eval_framework.tasks.base.Language attribute) HOL (eval_framework.tasks.base.Language attribute) HOM (eval_framework.tasks.base.Language attribute) HOO (eval_framework.tasks.base.Language attribute) HOP (eval_framework.tasks.base.Language attribute) HOR (eval_framework.tasks.base.Language attribute) HOS (eval_framework.tasks.base.Language attribute) HOT (eval_framework.tasks.base.Language attribute) HOV (eval_framework.tasks.base.Language attribute) HOW (eval_framework.tasks.base.Language attribute) HOY (eval_framework.tasks.base.Language attribute) HOZ (eval_framework.tasks.base.Language attribute) HPO (eval_framework.tasks.base.Language attribute) HPS (eval_framework.tasks.base.Language attribute) HRA (eval_framework.tasks.base.Language attribute) HRC (eval_framework.tasks.base.Language attribute) HRE (eval_framework.tasks.base.Language attribute) HRK (eval_framework.tasks.base.Language attribute) HRM (eval_framework.tasks.base.Language attribute) HRO (eval_framework.tasks.base.Language attribute) HRP (eval_framework.tasks.base.Language attribute) HRR (eval_framework.tasks.base.Language attribute) HRT (eval_framework.tasks.base.Language attribute) HRU (eval_framework.tasks.base.Language attribute) HRV (eval_framework.tasks.base.Language attribute) HRW (eval_framework.tasks.base.Language attribute) HRX (eval_framework.tasks.base.Language attribute) HRZ (eval_framework.tasks.base.Language attribute) HSB (eval_framework.tasks.base.Language attribute) HSF (eval_framework.tasks.base.Language attribute) HSH (eval_framework.tasks.base.Language attribute) HSL (eval_framework.tasks.base.Language attribute) HSN (eval_framework.tasks.base.Language attribute) HSS (eval_framework.tasks.base.Language attribute) HTI (eval_framework.tasks.base.Language attribute) HTO (eval_framework.tasks.base.Language attribute) HTS (eval_framework.tasks.base.Language attribute) HTU (eval_framework.tasks.base.Language attribute) HTX (eval_framework.tasks.base.Language attribute) HUB (eval_framework.tasks.base.Language attribute) HUC (eval_framework.tasks.base.Language attribute) HUD (eval_framework.tasks.base.Language attribute) HUE (eval_framework.tasks.base.Language attribute) HUF (eval_framework.tasks.base.Language attribute) HUG (eval_framework.tasks.base.Language attribute) HUH (eval_framework.tasks.base.Language attribute) HUI (eval_framework.tasks.base.Language attribute) HUJ (eval_framework.tasks.base.Language attribute) HUK (eval_framework.tasks.base.Language attribute) HUL (eval_framework.tasks.base.Language attribute) HUM (eval_framework.tasks.base.Language attribute) HumanEval (class in eval_framework.tasks.benchmarks.humaneval) HumanEvalInstruct (class in eval_framework.tasks.benchmarks.humaneval) HumanEvalMetricContext (class in eval_framework.tasks.benchmarks.humaneval) HUN (eval_framework.tasks.base.Language attribute) HUO (eval_framework.tasks.base.Language attribute) HUP (eval_framework.tasks.base.Language attribute) HUQ (eval_framework.tasks.base.Language attribute) HUR (eval_framework.tasks.base.Language attribute) HUS (eval_framework.tasks.base.Language attribute) HUT (eval_framework.tasks.base.Language attribute) HUU (eval_framework.tasks.base.Language attribute) HUV (eval_framework.tasks.base.Language attribute) HUW (eval_framework.tasks.base.Language attribute) HUX (eval_framework.tasks.base.Language attribute) HUY (eval_framework.tasks.base.Language attribute) HUZ (eval_framework.tasks.base.Language attribute) HVA (eval_framework.tasks.base.Language attribute) HVC (eval_framework.tasks.base.Language attribute) HVE (eval_framework.tasks.base.Language attribute) HVK (eval_framework.tasks.base.Language attribute) HVN (eval_framework.tasks.base.Language attribute) HVV (eval_framework.tasks.base.Language attribute) HWA (eval_framework.tasks.base.Language attribute) HWC (eval_framework.tasks.base.Language attribute) HWO (eval_framework.tasks.base.Language attribute) HYA (eval_framework.tasks.base.Language attribute) HYE (eval_framework.tasks.base.Language attribute) Hyperparameters (class in eval_framework.context.determined) HYW (eval_framework.tasks.base.Language attribute) I IAI (eval_framework.tasks.base.Language attribute) IAN (eval_framework.tasks.base.Language attribute) IAP (eval_framework.tasks.base.Language attribute) IAR (eval_framework.tasks.base.Language attribute) IBA (eval_framework.tasks.base.Language attribute) IBB (eval_framework.tasks.base.Language attribute) IBD (eval_framework.tasks.base.Language attribute) IBE (eval_framework.tasks.base.Language attribute) IBG (eval_framework.tasks.base.Language attribute) IBH (eval_framework.tasks.base.Language attribute) IBI (eval_framework.tasks.base.Language attribute) IBL (eval_framework.tasks.base.Language attribute) IBM (eval_framework.tasks.base.Language attribute) IBN (eval_framework.tasks.base.Language attribute) IBO (eval_framework.tasks.base.Language attribute) IBR (eval_framework.tasks.base.Language attribute) IBU (eval_framework.tasks.base.Language attribute) IBY (eval_framework.tasks.base.Language attribute) ICA (eval_framework.tasks.base.Language attribute) ICH (eval_framework.tasks.base.Language attribute) ICL (eval_framework.tasks.base.Language attribute) ICR (eval_framework.tasks.base.Language attribute) id (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.base.Sample attribute) IDA (eval_framework.tasks.base.Language attribute) IDB (eval_framework.tasks.base.Language attribute) IDC (eval_framework.tasks.base.Language attribute) IDD (eval_framework.tasks.base.Language attribute) IDE (eval_framework.tasks.base.Language attribute) IDI (eval_framework.tasks.base.Language attribute) IDO (eval_framework.tasks.base.Language attribute) IDR (eval_framework.tasks.base.Language attribute) IDS (eval_framework.tasks.base.Language attribute) IDT (eval_framework.tasks.base.Language attribute) IDU (eval_framework.tasks.base.Language attribute) IFA (eval_framework.tasks.base.Language attribute) IFB (eval_framework.tasks.base.Language attribute) IFE (eval_framework.tasks.base.Language attribute) IFEval (class in eval_framework.tasks.benchmarks.ifeval) IFEvalDe (class in eval_framework.tasks.benchmarks.ifeval) IFEvalFiSv (class in eval_framework.tasks.benchmarks.ifeval) IFEvalMetric (class in eval_framework.metrics.completion.ifeval) IFEvalMetricContext (class in eval_framework.metrics.completion.ifeval) IFF (eval_framework.tasks.base.Language attribute) IFK (eval_framework.tasks.base.Language attribute) IFM (eval_framework.tasks.base.Language attribute) IFU (eval_framework.tasks.base.Language attribute) IFY (eval_framework.tasks.base.Language attribute) IGB (eval_framework.tasks.base.Language attribute) IGE (eval_framework.tasks.base.Language attribute) IGG (eval_framework.tasks.base.Language attribute) IGL (eval_framework.tasks.base.Language attribute) IGM (eval_framework.tasks.base.Language attribute) IGN (eval_framework.tasks.base.Language attribute) IGO (eval_framework.tasks.base.Language attribute) IGS (eval_framework.tasks.base.Language attribute) IGW (eval_framework.tasks.base.Language attribute) IHB (eval_framework.tasks.base.Language attribute) IHI (eval_framework.tasks.base.Language attribute) IHP (eval_framework.tasks.base.Language attribute) IHW (eval_framework.tasks.base.Language attribute) III (eval_framework.tasks.base.Language attribute) IIN (eval_framework.tasks.base.Language attribute) IJC (eval_framework.tasks.base.Language attribute) IJE (eval_framework.tasks.base.Language attribute) IJJ (eval_framework.tasks.base.Language attribute) IJN (eval_framework.tasks.base.Language attribute) IJS (eval_framework.tasks.base.Language attribute) IKE (eval_framework.tasks.base.Language attribute) IKH (eval_framework.tasks.base.Language attribute) IKI (eval_framework.tasks.base.Language attribute) IKK (eval_framework.tasks.base.Language attribute) IKL (eval_framework.tasks.base.Language attribute) IKO (eval_framework.tasks.base.Language attribute) IKP (eval_framework.tasks.base.Language attribute) IKR (eval_framework.tasks.base.Language attribute) IKS (eval_framework.tasks.base.Language attribute) IKT (eval_framework.tasks.base.Language attribute) IKU (eval_framework.tasks.base.Language attribute) IKV (eval_framework.tasks.base.Language attribute) IKW (eval_framework.tasks.base.Language attribute) IKX (eval_framework.tasks.base.Language attribute) IKZ (eval_framework.tasks.base.Language attribute) ILA (eval_framework.tasks.base.Language attribute) ILB (eval_framework.tasks.base.Language attribute) ILE (eval_framework.tasks.base.Language attribute) ILG (eval_framework.tasks.base.Language attribute) ILI (eval_framework.tasks.base.Language attribute) ILK (eval_framework.tasks.base.Language attribute) ILL (eval_framework.tasks.base.Language attribute) ILM (eval_framework.tasks.base.Language attribute) ILO (eval_framework.tasks.base.Language attribute) ILP (eval_framework.tasks.base.Language attribute) ILS (eval_framework.tasks.base.Language attribute) ILU (eval_framework.tasks.base.Language attribute) ILV (eval_framework.tasks.base.Language attribute) ILW (eval_framework.tasks.base.Language attribute) IMA (eval_framework.tasks.base.Language attribute) IME (eval_framework.tasks.base.Language attribute) IMI (eval_framework.tasks.base.Language attribute) IML (eval_framework.tasks.base.Language attribute) IMN (eval_framework.tasks.base.Language attribute) IMO (eval_framework.tasks.base.Language attribute) implicit (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) import_file() (in module eval_framework.tasks.task_loader) import_models() (in module eval_framework.context.eval) IMR (eval_framework.tasks.base.Language attribute) IMS (eval_framework.tasks.base.Language attribute) IMT (eval_framework.tasks.base.Language attribute) IMY (eval_framework.tasks.base.Language attribute) INA (eval_framework.tasks.base.Language attribute) INB (eval_framework.tasks.base.Language attribute) INCLUDE (class in eval_framework.tasks.benchmarks.include) IND (eval_framework.tasks.base.Language attribute) InfiniteBench (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_CodeDebug (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_CodeRun (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_EnDia (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_EnMC (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_EnQA (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_MathFind (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_RetrieveKV2 (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_RetrieveNumber (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBench_RetrievePassKey1 (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBenchCompletion (class in eval_framework.tasks.benchmarks.infinitebench) InfiniteBenchLoglikelihood (class in eval_framework.tasks.benchmarks.infinitebench) ING (eval_framework.tasks.base.Language attribute) INH (eval_framework.tasks.base.Language attribute) INJ (eval_framework.tasks.base.Language attribute) INL (eval_framework.tasks.base.Language attribute) INM (eval_framework.tasks.base.Language attribute) INN (eval_framework.tasks.base.Language attribute) INO (eval_framework.tasks.base.Language attribute) INP (eval_framework.tasks.base.Language attribute) INS (eval_framework.tasks.base.Language attribute) instruction_id_list (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) INT (eval_framework.tasks.base.Language attribute) INZ (eval_framework.tasks.base.Language attribute) IOR (eval_framework.tasks.base.Language attribute) IOU (eval_framework.tasks.base.Language attribute) IOW (eval_framework.tasks.base.Language attribute) IPI (eval_framework.tasks.base.Language attribute) IPK (eval_framework.tasks.base.Language attribute) IPO (eval_framework.tasks.base.Language attribute) IQU (eval_framework.tasks.base.Language attribute) IQW (eval_framework.tasks.base.Language attribute) IRE (eval_framework.tasks.base.Language attribute) IRH (eval_framework.tasks.base.Language attribute) IRI (eval_framework.tasks.base.Language attribute) IRK (eval_framework.tasks.base.Language attribute) IRN (eval_framework.tasks.base.Language attribute) IRR (eval_framework.tasks.base.Language attribute) IRU (eval_framework.tasks.base.Language attribute) IRX (eval_framework.tasks.base.Language attribute) IRY (eval_framework.tasks.base.Language attribute) is_column_count_respected (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) is_create_table_statement() (in module eval_framework.metrics.llm.llm_judge_sql) is_just_json (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) is_registered() (in module eval_framework.tasks.registry) is_separator_respected (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) is_valid_html() (in module eval_framework.metrics.completion.struct_eval_metrics) is_valid_json (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) ISA (eval_framework.tasks.base.Language attribute) ISC (eval_framework.tasks.base.Language attribute) ISD (eval_framework.tasks.base.Language attribute) ISE (eval_framework.tasks.base.Language attribute) ISG (eval_framework.tasks.base.Language attribute) ISH (eval_framework.tasks.base.Language attribute) ISI (eval_framework.tasks.base.Language attribute) ISK (eval_framework.tasks.base.Language attribute) ISL (eval_framework.tasks.base.Language attribute) ISM (eval_framework.tasks.base.Language attribute) ISN (eval_framework.tasks.base.Language attribute) ISO (eval_framework.tasks.base.Language attribute) ISR (eval_framework.tasks.base.Language attribute) IST (eval_framework.tasks.base.Language attribute) ISU (eval_framework.tasks.base.Language attribute) ISV (eval_framework.tasks.base.Language attribute) ITA (eval_framework.tasks.base.Language attribute) ITB (eval_framework.tasks.base.Language attribute) ITD (eval_framework.tasks.base.Language attribute) ITE (eval_framework.tasks.base.Language attribute) iterate_samples() (eval_framework.tasks.base.BaseTask method) ITI (eval_framework.tasks.base.Language attribute) ITK (eval_framework.tasks.base.Language attribute) ITL (eval_framework.tasks.base.Language attribute) ITM (eval_framework.tasks.base.Language attribute) ITO (eval_framework.tasks.base.Language attribute) ITR (eval_framework.tasks.base.Language attribute) ITS (eval_framework.tasks.base.Language attribute) ITT (eval_framework.tasks.base.Language attribute) ITU (eval_framework.tasks.base.Language attribute) ITV (eval_framework.tasks.base.Language attribute) ITW (eval_framework.tasks.base.Language attribute) ITX (eval_framework.tasks.base.Language attribute) ITY (eval_framework.tasks.base.Language attribute) ITZ (eval_framework.tasks.base.Language attribute) IUM (eval_framework.tasks.base.Language attribute) IVB (eval_framework.tasks.base.Language attribute) IVV (eval_framework.tasks.base.Language attribute) IWK (eval_framework.tasks.base.Language attribute) IWM (eval_framework.tasks.base.Language attribute) IWO (eval_framework.tasks.base.Language attribute) IWS (eval_framework.tasks.base.Language attribute) IXC (eval_framework.tasks.base.Language attribute) IXI (eval_framework.tasks.base.Language attribute) IXJ (eval_framework.tasks.base.Language attribute) IXL (eval_framework.tasks.base.Language attribute) IYA (eval_framework.tasks.base.Language attribute) IYO (eval_framework.tasks.base.Language attribute) IYX (eval_framework.tasks.base.Language attribute) IZH (eval_framework.tasks.base.Language attribute) IZI (eval_framework.tasks.base.Language attribute) IZM (eval_framework.tasks.base.Language attribute) IZR (eval_framework.tasks.base.Language attribute) IZZ (eval_framework.tasks.base.Language attribute) J JAA (eval_framework.tasks.base.Language attribute) JAB (eval_framework.tasks.base.Language attribute) JAC (eval_framework.tasks.base.Language attribute) JAD (eval_framework.tasks.base.Language attribute) JAE (eval_framework.tasks.base.Language attribute) JAF (eval_framework.tasks.base.Language attribute) JAH (eval_framework.tasks.base.Language attribute) JAI (eval_framework.tasks.base.Language attribute) JAJ (eval_framework.tasks.base.Language attribute) JAK (eval_framework.tasks.base.Language attribute) JAL (eval_framework.tasks.base.Language attribute) JAM (eval_framework.tasks.base.Language attribute) JAN (eval_framework.tasks.base.Language attribute) JAO (eval_framework.tasks.base.Language attribute) JAP (eval_framework.tasks.base.Language attribute) JAQ (eval_framework.tasks.base.Language attribute) JAR (eval_framework.tasks.base.Language attribute) JAS (eval_framework.tasks.base.Language attribute) JAT (eval_framework.tasks.base.Language attribute) JAU (eval_framework.tasks.base.Language attribute) JAV (eval_framework.tasks.base.Language attribute) JAX (eval_framework.tasks.base.Language attribute) JAY (eval_framework.tasks.base.Language attribute) JAZ (eval_framework.tasks.base.Language attribute) JBE (eval_framework.tasks.base.Language attribute) JBI (eval_framework.tasks.base.Language attribute) JBJ (eval_framework.tasks.base.Language attribute) JBK (eval_framework.tasks.base.Language attribute) JBM (eval_framework.tasks.base.Language attribute) JBN (eval_framework.tasks.base.Language attribute) JBO (eval_framework.tasks.base.Language attribute) JBR (eval_framework.tasks.base.Language attribute) JBT (eval_framework.tasks.base.Language attribute) JBU (eval_framework.tasks.base.Language attribute) JBW (eval_framework.tasks.base.Language attribute) JCS (eval_framework.tasks.base.Language attribute) JCT (eval_framework.tasks.base.Language attribute) JDA (eval_framework.tasks.base.Language attribute) JDG (eval_framework.tasks.base.Language attribute) JDT (eval_framework.tasks.base.Language attribute) JEB (eval_framework.tasks.base.Language attribute) JEE (eval_framework.tasks.base.Language attribute) JEG (eval_framework.tasks.base.Language attribute) JEH (eval_framework.tasks.base.Language attribute) JEI (eval_framework.tasks.base.Language attribute) JEK (eval_framework.tasks.base.Language attribute) JEL (eval_framework.tasks.base.Language attribute) JEN (eval_framework.tasks.base.Language attribute) JER (eval_framework.tasks.base.Language attribute) JET (eval_framework.tasks.base.Language attribute) JEU (eval_framework.tasks.base.Language attribute) JGB (eval_framework.tasks.base.Language attribute) JGE (eval_framework.tasks.base.Language attribute) JGK (eval_framework.tasks.base.Language attribute) JGO (eval_framework.tasks.base.Language attribute) JHI (eval_framework.tasks.base.Language attribute) JHS (eval_framework.tasks.base.Language attribute) JIA (eval_framework.tasks.base.Language attribute) JIB (eval_framework.tasks.base.Language attribute) JIC (eval_framework.tasks.base.Language attribute) JID (eval_framework.tasks.base.Language attribute) JIE (eval_framework.tasks.base.Language attribute) JIG (eval_framework.tasks.base.Language attribute) JIH (eval_framework.tasks.base.Language attribute) JII (eval_framework.tasks.base.Language attribute) JIL (eval_framework.tasks.base.Language attribute) JIM (eval_framework.tasks.base.Language attribute) JIO (eval_framework.tasks.base.Language attribute) JIQ (eval_framework.tasks.base.Language attribute) JIT (eval_framework.tasks.base.Language attribute) JIU (eval_framework.tasks.base.Language attribute) JIV (eval_framework.tasks.base.Language attribute) JIY (eval_framework.tasks.base.Language attribute) JJE (eval_framework.tasks.base.Language attribute) JJR (eval_framework.tasks.base.Language attribute) JKA (eval_framework.tasks.base.Language attribute) JKM (eval_framework.tasks.base.Language attribute) JKO (eval_framework.tasks.base.Language attribute) JKP (eval_framework.tasks.base.Language attribute) JKR (eval_framework.tasks.base.Language attribute) JKS (eval_framework.tasks.base.Language attribute) JKU (eval_framework.tasks.base.Language attribute) JLE (eval_framework.tasks.base.Language attribute) JLS (eval_framework.tasks.base.Language attribute) JMA (eval_framework.tasks.base.Language attribute) JMB (eval_framework.tasks.base.Language attribute) JMC (eval_framework.tasks.base.Language attribute) JMD (eval_framework.tasks.base.Language attribute) JMI (eval_framework.tasks.base.Language attribute) JML (eval_framework.tasks.base.Language attribute) JMN (eval_framework.tasks.base.Language attribute) JMR (eval_framework.tasks.base.Language attribute) JMS (eval_framework.tasks.base.Language attribute) JMW (eval_framework.tasks.base.Language attribute) JMX (eval_framework.tasks.base.Language attribute) JNA (eval_framework.tasks.base.Language attribute) JND (eval_framework.tasks.base.Language attribute) JNG (eval_framework.tasks.base.Language attribute) JNI (eval_framework.tasks.base.Language attribute) JNJ (eval_framework.tasks.base.Language attribute) JNL (eval_framework.tasks.base.Language attribute) JNS (eval_framework.tasks.base.Language attribute) JOB (eval_framework.tasks.base.Language attribute) JOD (eval_framework.tasks.base.Language attribute) JOG (eval_framework.tasks.base.Language attribute) JOR (eval_framework.tasks.base.Language attribute) JOS (eval_framework.tasks.base.Language attribute) JOW (eval_framework.tasks.base.Language attribute) JPA (eval_framework.tasks.base.Language attribute) JPN (eval_framework.tasks.base.Language attribute) JPR (eval_framework.tasks.base.Language attribute) JQR (eval_framework.tasks.base.Language attribute) JRA (eval_framework.tasks.base.Language attribute) JRB (eval_framework.tasks.base.Language attribute) JRR (eval_framework.tasks.base.Language attribute) JRT (eval_framework.tasks.base.Language attribute) JRU (eval_framework.tasks.base.Language attribute) JSL (eval_framework.tasks.base.Language attribute) json_parsing_error (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) JsonFormat (class in eval_framework.metrics.completion.json_format) JsonFormatEvaluation (class in eval_framework.metrics.completion.json_format) JUA (eval_framework.tasks.base.Language attribute) JUB (eval_framework.tasks.base.Language attribute) JUC (eval_framework.tasks.base.Language attribute) JUD (eval_framework.tasks.base.Language attribute) judge_model_args (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) judge_model_name (eval_framework.context.determined.TaskArgs attribute) JUH (eval_framework.tasks.base.Language attribute) JUI (eval_framework.tasks.base.Language attribute) JUK (eval_framework.tasks.base.Language attribute) JUL (eval_framework.tasks.base.Language attribute) JUM (eval_framework.tasks.base.Language attribute) JUN (eval_framework.tasks.base.Language attribute) JUO (eval_framework.tasks.base.Language attribute) JUP (eval_framework.tasks.base.Language attribute) JUR (eval_framework.tasks.base.Language attribute) JUS (eval_framework.tasks.base.Language attribute) JUT (eval_framework.tasks.base.Language attribute) JUU (eval_framework.tasks.base.Language attribute) JUW (eval_framework.tasks.base.Language attribute) JUY (eval_framework.tasks.base.Language attribute) JVD (eval_framework.tasks.base.Language attribute) JVN (eval_framework.tasks.base.Language attribute) JWI (eval_framework.tasks.base.Language attribute) JYA (eval_framework.tasks.base.Language attribute) JYE (eval_framework.tasks.base.Language attribute) JYY (eval_framework.tasks.base.Language attribute) K KAA (eval_framework.tasks.base.Language attribute) KAB (eval_framework.tasks.base.Language attribute) KAC (eval_framework.tasks.base.Language attribute) KAD (eval_framework.tasks.base.Language attribute) KAE (eval_framework.tasks.base.Language attribute) KAF (eval_framework.tasks.base.Language attribute) KAG (eval_framework.tasks.base.Language attribute) KAH (eval_framework.tasks.base.Language attribute) KAI (eval_framework.tasks.base.Language attribute) KAJ (eval_framework.tasks.base.Language attribute) KAK (eval_framework.tasks.base.Language attribute) KAL (eval_framework.tasks.base.Language attribute) KAM (eval_framework.tasks.base.Language attribute) KAN (eval_framework.tasks.base.Language attribute) KAO (eval_framework.tasks.base.Language attribute) KAP (eval_framework.tasks.base.Language attribute) KAQ (eval_framework.tasks.base.Language attribute) KAS (eval_framework.tasks.base.Language attribute) KAT (eval_framework.tasks.base.Language attribute) KAU (eval_framework.tasks.base.Language attribute) KAV (eval_framework.tasks.base.Language attribute) KAW (eval_framework.tasks.base.Language attribute) KAX (eval_framework.tasks.base.Language attribute) KAY (eval_framework.tasks.base.Language attribute) KAZ (eval_framework.tasks.base.Language attribute) KBA (eval_framework.tasks.base.Language attribute) KBB (eval_framework.tasks.base.Language attribute) KBC (eval_framework.tasks.base.Language attribute) KBD (eval_framework.tasks.base.Language attribute) KBE (eval_framework.tasks.base.Language attribute) KBF (eval_framework.tasks.base.Language attribute) KBG (eval_framework.tasks.base.Language attribute) KBH (eval_framework.tasks.base.Language attribute) KBI (eval_framework.tasks.base.Language attribute) KBJ (eval_framework.tasks.base.Language attribute) KBK (eval_framework.tasks.base.Language attribute) KBL (eval_framework.tasks.base.Language attribute) KBM (eval_framework.tasks.base.Language attribute) KBN (eval_framework.tasks.base.Language attribute) KBO (eval_framework.tasks.base.Language attribute) KBP (eval_framework.tasks.base.Language attribute) KBQ (eval_framework.tasks.base.Language attribute) KBR (eval_framework.tasks.base.Language attribute) KBS (eval_framework.tasks.base.Language attribute) KBT (eval_framework.tasks.base.Language attribute) KBU (eval_framework.tasks.base.Language attribute) KBV (eval_framework.tasks.base.Language attribute) KBW (eval_framework.tasks.base.Language attribute) KBX (eval_framework.tasks.base.Language attribute) KBY (eval_framework.tasks.base.Language attribute) KBZ (eval_framework.tasks.base.Language attribute) KCA (eval_framework.tasks.base.Language attribute) KCB (eval_framework.tasks.base.Language attribute) KCC (eval_framework.tasks.base.Language attribute) KCD (eval_framework.tasks.base.Language attribute) KCE (eval_framework.tasks.base.Language attribute) KCF (eval_framework.tasks.base.Language attribute) KCG (eval_framework.tasks.base.Language attribute) KCH (eval_framework.tasks.base.Language attribute) KCI (eval_framework.tasks.base.Language attribute) KCJ (eval_framework.tasks.base.Language attribute) KCK (eval_framework.tasks.base.Language attribute) KCL (eval_framework.tasks.base.Language attribute) KCM (eval_framework.tasks.base.Language attribute) KCN (eval_framework.tasks.base.Language attribute) KCO (eval_framework.tasks.base.Language attribute) KCP (eval_framework.tasks.base.Language attribute) KCQ (eval_framework.tasks.base.Language attribute) KCR (eval_framework.tasks.base.Language attribute) KCS (eval_framework.tasks.base.Language attribute) KCT (eval_framework.tasks.base.Language attribute) KCU (eval_framework.tasks.base.Language attribute) KCV (eval_framework.tasks.base.Language attribute) KCW (eval_framework.tasks.base.Language attribute) KCX (eval_framework.tasks.base.Language attribute) KCY (eval_framework.tasks.base.Language attribute) KCZ (eval_framework.tasks.base.Language attribute) KDA (eval_framework.tasks.base.Language attribute) KDC (eval_framework.tasks.base.Language attribute) KDD (eval_framework.tasks.base.Language attribute) KDE (eval_framework.tasks.base.Language attribute) KDF (eval_framework.tasks.base.Language attribute) KDG (eval_framework.tasks.base.Language attribute) KDH (eval_framework.tasks.base.Language attribute) KDI (eval_framework.tasks.base.Language attribute) KDJ (eval_framework.tasks.base.Language attribute) KDK (eval_framework.tasks.base.Language attribute) KDL (eval_framework.tasks.base.Language attribute) KDM (eval_framework.tasks.base.Language attribute) KDN (eval_framework.tasks.base.Language attribute) KDP (eval_framework.tasks.base.Language attribute) KDQ (eval_framework.tasks.base.Language attribute) KDR (eval_framework.tasks.base.Language attribute) KDS (eval_framework.tasks.base.Language attribute) KDT (eval_framework.tasks.base.Language attribute) KDU (eval_framework.tasks.base.Language attribute) KDV (eval_framework.tasks.base.Language attribute) KDW (eval_framework.tasks.base.Language attribute) KDX (eval_framework.tasks.base.Language attribute) KDY (eval_framework.tasks.base.Language attribute) KDZ (eval_framework.tasks.base.Language attribute) KEA (eval_framework.tasks.base.Language attribute) KEB (eval_framework.tasks.base.Language attribute) KEC (eval_framework.tasks.base.Language attribute) KED (eval_framework.tasks.base.Language attribute) KEE (eval_framework.tasks.base.Language attribute) KEF (eval_framework.tasks.base.Language attribute) KEG (eval_framework.tasks.base.Language attribute) KEH (eval_framework.tasks.base.Language attribute) KEI (eval_framework.tasks.base.Language attribute) KEJ (eval_framework.tasks.base.Language attribute) KEK (eval_framework.tasks.base.Language attribute) KEL (eval_framework.tasks.base.Language attribute) KEM (eval_framework.tasks.base.Language attribute) KEN (eval_framework.tasks.base.Language attribute) KEO (eval_framework.tasks.base.Language attribute) KEP (eval_framework.tasks.base.Language attribute) KEQ (eval_framework.tasks.base.Language attribute) KER (eval_framework.tasks.base.Language attribute) KES (eval_framework.tasks.base.Language attribute) KET (eval_framework.tasks.base.Language attribute) KEU (eval_framework.tasks.base.Language attribute) KEV (eval_framework.tasks.base.Language attribute) KEW (eval_framework.tasks.base.Language attribute) KEX (eval_framework.tasks.base.Language attribute) key (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) (eval_framework.result_processors.base.Result attribute) KEY (eval_framework.tasks.base.Language attribute) KEYS (eval_framework.metrics.base.BaseMetric attribute) (eval_framework.metrics.completion.csv_format.CSVFormat attribute) (eval_framework.metrics.llm.llm_judge_coherence.LLMJudgeCoherence attribute) (eval_framework.metrics.llm.llm_judge_instruction.LLMJudgeInstruction attribute) keywords (eval_framework.metrics.completion.struct_eval_metrics.RenderableStructMetricContext attribute) KEZ (eval_framework.tasks.base.Language attribute) KFA (eval_framework.tasks.base.Language attribute) KFB (eval_framework.tasks.base.Language attribute) KFC (eval_framework.tasks.base.Language attribute) KFD (eval_framework.tasks.base.Language attribute) KFE (eval_framework.tasks.base.Language attribute) KFF (eval_framework.tasks.base.Language attribute) KFG (eval_framework.tasks.base.Language attribute) KFH (eval_framework.tasks.base.Language attribute) KFI (eval_framework.tasks.base.Language attribute) KFJ (eval_framework.tasks.base.Language attribute) KFK (eval_framework.tasks.base.Language attribute) KFL (eval_framework.tasks.base.Language attribute) KFM (eval_framework.tasks.base.Language attribute) KFN (eval_framework.tasks.base.Language attribute) KFO (eval_framework.tasks.base.Language attribute) KFP (eval_framework.tasks.base.Language attribute) KFQ (eval_framework.tasks.base.Language attribute) KFR (eval_framework.tasks.base.Language attribute) KFS (eval_framework.tasks.base.Language attribute) KFT (eval_framework.tasks.base.Language attribute) KFU (eval_framework.tasks.base.Language attribute) KFV (eval_framework.tasks.base.Language attribute) KFW (eval_framework.tasks.base.Language attribute) KFX (eval_framework.tasks.base.Language attribute) KFY (eval_framework.tasks.base.Language attribute) KFZ (eval_framework.tasks.base.Language attribute) KGA (eval_framework.tasks.base.Language attribute) KGB (eval_framework.tasks.base.Language attribute) KGC (eval_framework.tasks.base.Language attribute) KGD (eval_framework.tasks.base.Language attribute) KGE (eval_framework.tasks.base.Language attribute) KGF (eval_framework.tasks.base.Language attribute) KGG (eval_framework.tasks.base.Language attribute) KGH (eval_framework.tasks.base.Language attribute) KGI (eval_framework.tasks.base.Language attribute) KGJ (eval_framework.tasks.base.Language attribute) KGK (eval_framework.tasks.base.Language attribute) KGL (eval_framework.tasks.base.Language attribute) KGM (eval_framework.tasks.base.Language attribute) KGN (eval_framework.tasks.base.Language attribute) KGO (eval_framework.tasks.base.Language attribute) KGP (eval_framework.tasks.base.Language attribute) KGQ (eval_framework.tasks.base.Language attribute) KGR (eval_framework.tasks.base.Language attribute) KGS (eval_framework.tasks.base.Language attribute) KGT (eval_framework.tasks.base.Language attribute) KGU (eval_framework.tasks.base.Language attribute) KGV (eval_framework.tasks.base.Language attribute) KGW (eval_framework.tasks.base.Language attribute) KGX (eval_framework.tasks.base.Language attribute) KGY (eval_framework.tasks.base.Language attribute) KHA (eval_framework.tasks.base.Language attribute) KHB (eval_framework.tasks.base.Language attribute) KHC (eval_framework.tasks.base.Language attribute) KHD (eval_framework.tasks.base.Language attribute) KHE (eval_framework.tasks.base.Language attribute) KHF (eval_framework.tasks.base.Language attribute) KHG (eval_framework.tasks.base.Language attribute) KHH (eval_framework.tasks.base.Language attribute) KHJ (eval_framework.tasks.base.Language attribute) KHK (eval_framework.tasks.base.Language attribute) KHL (eval_framework.tasks.base.Language attribute) KHM (eval_framework.tasks.base.Language attribute) KHN (eval_framework.tasks.base.Language attribute) KHO (eval_framework.tasks.base.Language attribute) KHP (eval_framework.tasks.base.Language attribute) KHQ (eval_framework.tasks.base.Language attribute) KHR (eval_framework.tasks.base.Language attribute) KHS (eval_framework.tasks.base.Language attribute) KHT (eval_framework.tasks.base.Language attribute) KHU (eval_framework.tasks.base.Language attribute) KHV (eval_framework.tasks.base.Language attribute) KHW (eval_framework.tasks.base.Language attribute) KHX (eval_framework.tasks.base.Language attribute) KHY (eval_framework.tasks.base.Language attribute) KHZ (eval_framework.tasks.base.Language attribute) KIA (eval_framework.tasks.base.Language attribute) KIB (eval_framework.tasks.base.Language attribute) KIC (eval_framework.tasks.base.Language attribute) KID (eval_framework.tasks.base.Language attribute) KIE (eval_framework.tasks.base.Language attribute) KIF (eval_framework.tasks.base.Language attribute) KIG (eval_framework.tasks.base.Language attribute) KIH (eval_framework.tasks.base.Language attribute) KII (eval_framework.tasks.base.Language attribute) KIJ (eval_framework.tasks.base.Language attribute) KIK (eval_framework.tasks.base.Language attribute) KIL (eval_framework.tasks.base.Language attribute) KIM (eval_framework.tasks.base.Language attribute) KIN (eval_framework.tasks.base.Language attribute) KIO (eval_framework.tasks.base.Language attribute) KIP (eval_framework.tasks.base.Language attribute) KIQ (eval_framework.tasks.base.Language attribute) KIR (eval_framework.tasks.base.Language attribute) KIS (eval_framework.tasks.base.Language attribute) KIT (eval_framework.tasks.base.Language attribute) KIU (eval_framework.tasks.base.Language attribute) KIV (eval_framework.tasks.base.Language attribute) KIW (eval_framework.tasks.base.Language attribute) KIX (eval_framework.tasks.base.Language attribute) KIY (eval_framework.tasks.base.Language attribute) KIZ (eval_framework.tasks.base.Language attribute) KJA (eval_framework.tasks.base.Language attribute) KJB (eval_framework.tasks.base.Language attribute) KJC (eval_framework.tasks.base.Language attribute) KJD (eval_framework.tasks.base.Language attribute) KJE (eval_framework.tasks.base.Language attribute) KJF (eval_framework.tasks.base.Language attribute) KJG (eval_framework.tasks.base.Language attribute) KJH (eval_framework.tasks.base.Language attribute) KJI (eval_framework.tasks.base.Language attribute) KJJ (eval_framework.tasks.base.Language attribute) KJK (eval_framework.tasks.base.Language attribute) KJL (eval_framework.tasks.base.Language attribute) KJM (eval_framework.tasks.base.Language attribute) KJN (eval_framework.tasks.base.Language attribute) KJO (eval_framework.tasks.base.Language attribute) KJP (eval_framework.tasks.base.Language attribute) KJQ (eval_framework.tasks.base.Language attribute) KJR (eval_framework.tasks.base.Language attribute) KJS (eval_framework.tasks.base.Language attribute) KJT (eval_framework.tasks.base.Language attribute) KJU (eval_framework.tasks.base.Language attribute) KJV (eval_framework.tasks.base.Language attribute) KJX (eval_framework.tasks.base.Language attribute) KJY (eval_framework.tasks.base.Language attribute) KJZ (eval_framework.tasks.base.Language attribute) KKA (eval_framework.tasks.base.Language attribute) KKB (eval_framework.tasks.base.Language attribute) KKC (eval_framework.tasks.base.Language attribute) KKD (eval_framework.tasks.base.Language attribute) KKE (eval_framework.tasks.base.Language attribute) KKF (eval_framework.tasks.base.Language attribute) KKG (eval_framework.tasks.base.Language attribute) KKH (eval_framework.tasks.base.Language attribute) KKI (eval_framework.tasks.base.Language attribute) KKJ (eval_framework.tasks.base.Language attribute) KKK (eval_framework.tasks.base.Language attribute) KKL (eval_framework.tasks.base.Language attribute) KKM (eval_framework.tasks.base.Language attribute) KKN (eval_framework.tasks.base.Language attribute) KKO (eval_framework.tasks.base.Language attribute) KKP (eval_framework.tasks.base.Language attribute) KKQ (eval_framework.tasks.base.Language attribute) KKR (eval_framework.tasks.base.Language attribute) KKS (eval_framework.tasks.base.Language attribute) KKT (eval_framework.tasks.base.Language attribute) KKU (eval_framework.tasks.base.Language attribute) KKV (eval_framework.tasks.base.Language attribute) KKW (eval_framework.tasks.base.Language attribute) KKX (eval_framework.tasks.base.Language attribute) KKY (eval_framework.tasks.base.Language attribute) KKZ (eval_framework.tasks.base.Language attribute) KLA (eval_framework.tasks.base.Language attribute) KLB (eval_framework.tasks.base.Language attribute) KLC (eval_framework.tasks.base.Language attribute) KLD (eval_framework.tasks.base.Language attribute) KLE (eval_framework.tasks.base.Language attribute) KLF (eval_framework.tasks.base.Language attribute) KLG (eval_framework.tasks.base.Language attribute) KLH (eval_framework.tasks.base.Language attribute) KLI (eval_framework.tasks.base.Language attribute) KLJ (eval_framework.tasks.base.Language attribute) KLK (eval_framework.tasks.base.Language attribute) KLL (eval_framework.tasks.base.Language attribute) KLM (eval_framework.tasks.base.Language attribute) KLN (eval_framework.tasks.base.Language attribute) KLO (eval_framework.tasks.base.Language attribute) KLP (eval_framework.tasks.base.Language attribute) KLQ (eval_framework.tasks.base.Language attribute) KLR (eval_framework.tasks.base.Language attribute) KLS (eval_framework.tasks.base.Language attribute) KLT (eval_framework.tasks.base.Language attribute) KLU (eval_framework.tasks.base.Language attribute) KLV (eval_framework.tasks.base.Language attribute) KLW (eval_framework.tasks.base.Language attribute) KLX (eval_framework.tasks.base.Language attribute) KLY (eval_framework.tasks.base.Language attribute) KLZ (eval_framework.tasks.base.Language attribute) KMA (eval_framework.tasks.base.Language attribute) KMB (eval_framework.tasks.base.Language attribute) KMC (eval_framework.tasks.base.Language attribute) KMD (eval_framework.tasks.base.Language attribute) KME (eval_framework.tasks.base.Language attribute) KMF (eval_framework.tasks.base.Language attribute) KMG (eval_framework.tasks.base.Language attribute) KMH (eval_framework.tasks.base.Language attribute) KMI (eval_framework.tasks.base.Language attribute) KMJ (eval_framework.tasks.base.Language attribute) KMK (eval_framework.tasks.base.Language attribute) KML (eval_framework.tasks.base.Language attribute) KMM (eval_framework.tasks.base.Language attribute) KMN (eval_framework.tasks.base.Language attribute) KMO (eval_framework.tasks.base.Language attribute) KMP (eval_framework.tasks.base.Language attribute) KMQ (eval_framework.tasks.base.Language attribute) KMR (eval_framework.tasks.base.Language attribute) KMS (eval_framework.tasks.base.Language attribute) KMT (eval_framework.tasks.base.Language attribute) KMU (eval_framework.tasks.base.Language attribute) KMV (eval_framework.tasks.base.Language attribute) KMW (eval_framework.tasks.base.Language attribute) KMX (eval_framework.tasks.base.Language attribute) KMY (eval_framework.tasks.base.Language attribute) KMZ (eval_framework.tasks.base.Language attribute) KNA (eval_framework.tasks.base.Language attribute) KNB (eval_framework.tasks.base.Language attribute) KNC (eval_framework.tasks.base.Language attribute) KND (eval_framework.tasks.base.Language attribute) KNE (eval_framework.tasks.base.Language attribute) KNF (eval_framework.tasks.base.Language attribute) KNG (eval_framework.tasks.base.Language attribute) KNH (eval_framework.tasks.base.Language attribute) KNI (eval_framework.tasks.base.Language attribute) KNJ (eval_framework.tasks.base.Language attribute) KNK (eval_framework.tasks.base.Language attribute) KNL (eval_framework.tasks.base.Language attribute) KNM (eval_framework.tasks.base.Language attribute) KNN (eval_framework.tasks.base.Language attribute) KNO (eval_framework.tasks.base.Language attribute) KNP (eval_framework.tasks.base.Language attribute) KNQ (eval_framework.tasks.base.Language attribute) KNR (eval_framework.tasks.base.Language attribute) KNS (eval_framework.tasks.base.Language attribute) KNT (eval_framework.tasks.base.Language attribute) KNU (eval_framework.tasks.base.Language attribute) KNV (eval_framework.tasks.base.Language attribute) KNW (eval_framework.tasks.base.Language attribute) KNX (eval_framework.tasks.base.Language attribute) KNY (eval_framework.tasks.base.Language attribute) KNZ (eval_framework.tasks.base.Language attribute) KOA (eval_framework.tasks.base.Language attribute) KOB (eval_framework.tasks.base.Language attribute) KOC (eval_framework.tasks.base.Language attribute) KOD (eval_framework.tasks.base.Language attribute) KOE (eval_framework.tasks.base.Language attribute) KOF (eval_framework.tasks.base.Language attribute) KOG (eval_framework.tasks.base.Language attribute) KOH (eval_framework.tasks.base.Language attribute) KOI (eval_framework.tasks.base.Language attribute) KOJ (eval_framework.tasks.base.Language attribute) KOK (eval_framework.tasks.base.Language attribute) KOL (eval_framework.tasks.base.Language attribute) KOM (eval_framework.tasks.base.Language attribute) KON (eval_framework.tasks.base.Language attribute) KOO (eval_framework.tasks.base.Language attribute) KOP (eval_framework.tasks.base.Language attribute) KOQ (eval_framework.tasks.base.Language attribute) KOR (eval_framework.tasks.base.Language attribute) KOS (eval_framework.tasks.base.Language attribute) KOT (eval_framework.tasks.base.Language attribute) KOU (eval_framework.tasks.base.Language attribute) KOV (eval_framework.tasks.base.Language attribute) KOW (eval_framework.tasks.base.Language attribute) KOX (eval_framework.tasks.base.Language attribute) KOY (eval_framework.tasks.base.Language attribute) KOZ (eval_framework.tasks.base.Language attribute) KPA (eval_framework.tasks.base.Language attribute) KPB (eval_framework.tasks.base.Language attribute) KPC (eval_framework.tasks.base.Language attribute) KPD (eval_framework.tasks.base.Language attribute) KPE (eval_framework.tasks.base.Language attribute) KPF (eval_framework.tasks.base.Language attribute) KPG (eval_framework.tasks.base.Language attribute) KPH (eval_framework.tasks.base.Language attribute) KPI (eval_framework.tasks.base.Language attribute) KPJ (eval_framework.tasks.base.Language attribute) KPK (eval_framework.tasks.base.Language attribute) KPL (eval_framework.tasks.base.Language attribute) KPM (eval_framework.tasks.base.Language attribute) KPN (eval_framework.tasks.base.Language attribute) KPO (eval_framework.tasks.base.Language attribute) KPP (eval_framework.tasks.base.Language attribute) KPQ (eval_framework.tasks.base.Language attribute) KPR (eval_framework.tasks.base.Language attribute) KPS (eval_framework.tasks.base.Language attribute) KPT (eval_framework.tasks.base.Language attribute) KPU (eval_framework.tasks.base.Language attribute) KPV (eval_framework.tasks.base.Language attribute) KPW (eval_framework.tasks.base.Language attribute) KPX (eval_framework.tasks.base.Language attribute) KPY (eval_framework.tasks.base.Language attribute) KPZ (eval_framework.tasks.base.Language attribute) KQA (eval_framework.tasks.base.Language attribute) KQB (eval_framework.tasks.base.Language attribute) KQC (eval_framework.tasks.base.Language attribute) KQD (eval_framework.tasks.base.Language attribute) KQE (eval_framework.tasks.base.Language attribute) KQF (eval_framework.tasks.base.Language attribute) KQG (eval_framework.tasks.base.Language attribute) KQH (eval_framework.tasks.base.Language attribute) KQI (eval_framework.tasks.base.Language attribute) KQJ (eval_framework.tasks.base.Language attribute) KQK (eval_framework.tasks.base.Language attribute) KQL (eval_framework.tasks.base.Language attribute) KQM (eval_framework.tasks.base.Language attribute) KQN (eval_framework.tasks.base.Language attribute) KQO (eval_framework.tasks.base.Language attribute) KQP (eval_framework.tasks.base.Language attribute) KQQ (eval_framework.tasks.base.Language attribute) KQR (eval_framework.tasks.base.Language attribute) KQS (eval_framework.tasks.base.Language attribute) KQT (eval_framework.tasks.base.Language attribute) KQU (eval_framework.tasks.base.Language attribute) KQV (eval_framework.tasks.base.Language attribute) KQW (eval_framework.tasks.base.Language attribute) KQX (eval_framework.tasks.base.Language attribute) KQY (eval_framework.tasks.base.Language attribute) KQZ (eval_framework.tasks.base.Language attribute) KRA (eval_framework.tasks.base.Language attribute) KRB (eval_framework.tasks.base.Language attribute) KRC (eval_framework.tasks.base.Language attribute) KRD (eval_framework.tasks.base.Language attribute) KRE (eval_framework.tasks.base.Language attribute) KRF (eval_framework.tasks.base.Language attribute) KRG (eval_framework.tasks.base.Language attribute) KRH (eval_framework.tasks.base.Language attribute) KRI (eval_framework.tasks.base.Language attribute) KRJ (eval_framework.tasks.base.Language attribute) KRK (eval_framework.tasks.base.Language attribute) KRL (eval_framework.tasks.base.Language attribute) KRM (eval_framework.tasks.base.Language attribute) KRN (eval_framework.tasks.base.Language attribute) KRP (eval_framework.tasks.base.Language attribute) KRQ (eval_framework.tasks.base.Language attribute) KRR (eval_framework.tasks.base.Language attribute) KRS (eval_framework.tasks.base.Language attribute) KRT (eval_framework.tasks.base.Language attribute) KRU (eval_framework.tasks.base.Language attribute) KRV (eval_framework.tasks.base.Language attribute) KRW (eval_framework.tasks.base.Language attribute) KRX (eval_framework.tasks.base.Language attribute) KRY (eval_framework.tasks.base.Language attribute) KRZ (eval_framework.tasks.base.Language attribute) KSA (eval_framework.tasks.base.Language attribute) KSB (eval_framework.tasks.base.Language attribute) KSC (eval_framework.tasks.base.Language attribute) KSD (eval_framework.tasks.base.Language attribute) KSE (eval_framework.tasks.base.Language attribute) KSF (eval_framework.tasks.base.Language attribute) KSG (eval_framework.tasks.base.Language attribute) KSH (eval_framework.tasks.base.Language attribute) KSI (eval_framework.tasks.base.Language attribute) KSJ (eval_framework.tasks.base.Language attribute) KSK (eval_framework.tasks.base.Language attribute) KSL (eval_framework.tasks.base.Language attribute) KSM (eval_framework.tasks.base.Language attribute) KSN (eval_framework.tasks.base.Language attribute) KSO (eval_framework.tasks.base.Language attribute) KSP (eval_framework.tasks.base.Language attribute) KSQ (eval_framework.tasks.base.Language attribute) KSR (eval_framework.tasks.base.Language attribute) KSS (eval_framework.tasks.base.Language attribute) KST (eval_framework.tasks.base.Language attribute) KSU (eval_framework.tasks.base.Language attribute) KSV (eval_framework.tasks.base.Language attribute) KSW (eval_framework.tasks.base.Language attribute) KSX (eval_framework.tasks.base.Language attribute) KSY (eval_framework.tasks.base.Language attribute) KSZ (eval_framework.tasks.base.Language attribute) KTA (eval_framework.tasks.base.Language attribute) KTB (eval_framework.tasks.base.Language attribute) KTC (eval_framework.tasks.base.Language attribute) KTD (eval_framework.tasks.base.Language attribute) KTE (eval_framework.tasks.base.Language attribute) KTF (eval_framework.tasks.base.Language attribute) KTG (eval_framework.tasks.base.Language attribute) KTH (eval_framework.tasks.base.Language attribute) KTI (eval_framework.tasks.base.Language attribute) KTJ (eval_framework.tasks.base.Language attribute) KTK (eval_framework.tasks.base.Language attribute) KTL (eval_framework.tasks.base.Language attribute) KTM (eval_framework.tasks.base.Language attribute) KTN (eval_framework.tasks.base.Language attribute) KTO (eval_framework.tasks.base.Language attribute) KTP (eval_framework.tasks.base.Language attribute) KTQ (eval_framework.tasks.base.Language attribute) KTR (eval_framework.tasks.base.Language attribute) KTS (eval_framework.tasks.base.Language attribute) KTT (eval_framework.tasks.base.Language attribute) KTU (eval_framework.tasks.base.Language attribute) KTV (eval_framework.tasks.base.Language attribute) KTW (eval_framework.tasks.base.Language attribute) KTX (eval_framework.tasks.base.Language attribute) KTY (eval_framework.tasks.base.Language attribute) KTZ (eval_framework.tasks.base.Language attribute) KUA (eval_framework.tasks.base.Language attribute) KUB (eval_framework.tasks.base.Language attribute) KUC (eval_framework.tasks.base.Language attribute) KUD (eval_framework.tasks.base.Language attribute) KUE (eval_framework.tasks.base.Language attribute) KUF (eval_framework.tasks.base.Language attribute) KUG (eval_framework.tasks.base.Language attribute) KUH (eval_framework.tasks.base.Language attribute) KUI (eval_framework.tasks.base.Language attribute) KUJ (eval_framework.tasks.base.Language attribute) KUK (eval_framework.tasks.base.Language attribute) KUL (eval_framework.tasks.base.Language attribute) KUM (eval_framework.tasks.base.Language attribute) KUN (eval_framework.tasks.base.Language attribute) KUO (eval_framework.tasks.base.Language attribute) KUP (eval_framework.tasks.base.Language attribute) KUQ (eval_framework.tasks.base.Language attribute) KUR (eval_framework.tasks.base.Language attribute) KUS (eval_framework.tasks.base.Language attribute) KUT (eval_framework.tasks.base.Language attribute) KUU (eval_framework.tasks.base.Language attribute) KUV (eval_framework.tasks.base.Language attribute) KUW (eval_framework.tasks.base.Language attribute) KUX (eval_framework.tasks.base.Language attribute) KUY (eval_framework.tasks.base.Language attribute) KUZ (eval_framework.tasks.base.Language attribute) KVA (eval_framework.tasks.base.Language attribute) KVB (eval_framework.tasks.base.Language attribute) KVC (eval_framework.tasks.base.Language attribute) KVD (eval_framework.tasks.base.Language attribute) KVE (eval_framework.tasks.base.Language attribute) KVF (eval_framework.tasks.base.Language attribute) KVG (eval_framework.tasks.base.Language attribute) KVH (eval_framework.tasks.base.Language attribute) KVI (eval_framework.tasks.base.Language attribute) KVJ (eval_framework.tasks.base.Language attribute) KVK (eval_framework.tasks.base.Language attribute) KVL (eval_framework.tasks.base.Language attribute) KVM (eval_framework.tasks.base.Language attribute) KVN (eval_framework.tasks.base.Language attribute) KVO (eval_framework.tasks.base.Language attribute) KVP (eval_framework.tasks.base.Language attribute) KVQ (eval_framework.tasks.base.Language attribute) KVR (eval_framework.tasks.base.Language attribute) KVS (eval_framework.tasks.base.Language attribute) KVT (eval_framework.tasks.base.Language attribute) KVU (eval_framework.tasks.base.Language attribute) KVV (eval_framework.tasks.base.Language attribute) KVW (eval_framework.tasks.base.Language attribute) KVX (eval_framework.tasks.base.Language attribute) KVY (eval_framework.tasks.base.Language attribute) KVZ (eval_framework.tasks.base.Language attribute) KWA (eval_framework.tasks.base.Language attribute) KWB (eval_framework.tasks.base.Language attribute) KWC (eval_framework.tasks.base.Language attribute) KWD (eval_framework.tasks.base.Language attribute) KWE (eval_framework.tasks.base.Language attribute) KWF (eval_framework.tasks.base.Language attribute) KWG (eval_framework.tasks.base.Language attribute) KWH (eval_framework.tasks.base.Language attribute) KWI (eval_framework.tasks.base.Language attribute) KWJ (eval_framework.tasks.base.Language attribute) KWK (eval_framework.tasks.base.Language attribute) KWL (eval_framework.tasks.base.Language attribute) KWM (eval_framework.tasks.base.Language attribute) KWN (eval_framework.tasks.base.Language attribute) KWO (eval_framework.tasks.base.Language attribute) KWP (eval_framework.tasks.base.Language attribute) KWQ (eval_framework.tasks.base.Language attribute) KWR (eval_framework.tasks.base.Language attribute) KWS (eval_framework.tasks.base.Language attribute) KWT (eval_framework.tasks.base.Language attribute) KWU (eval_framework.tasks.base.Language attribute) KWV (eval_framework.tasks.base.Language attribute) KWW (eval_framework.tasks.base.Language attribute) KWX (eval_framework.tasks.base.Language attribute) KWY (eval_framework.tasks.base.Language attribute) KWZ (eval_framework.tasks.base.Language attribute) KXA (eval_framework.tasks.base.Language attribute) KXB (eval_framework.tasks.base.Language attribute) KXC (eval_framework.tasks.base.Language attribute) KXD (eval_framework.tasks.base.Language attribute) KXE (eval_framework.tasks.base.Language attribute) KXF (eval_framework.tasks.base.Language attribute) KXG (eval_framework.tasks.base.Language attribute) KXH (eval_framework.tasks.base.Language attribute) KXI (eval_framework.tasks.base.Language attribute) KXJ (eval_framework.tasks.base.Language attribute) KXK (eval_framework.tasks.base.Language attribute) KXL (eval_framework.tasks.base.Language attribute) KXM (eval_framework.tasks.base.Language attribute) KXN (eval_framework.tasks.base.Language attribute) KXO (eval_framework.tasks.base.Language attribute) KXP (eval_framework.tasks.base.Language attribute) KXQ (eval_framework.tasks.base.Language attribute) KXR (eval_framework.tasks.base.Language attribute) KXS (eval_framework.tasks.base.Language attribute) KXT (eval_framework.tasks.base.Language attribute) KXU (eval_framework.tasks.base.Language attribute) KXV (eval_framework.tasks.base.Language attribute) KXW (eval_framework.tasks.base.Language attribute) KXX (eval_framework.tasks.base.Language attribute) KXY (eval_framework.tasks.base.Language attribute) KXZ (eval_framework.tasks.base.Language attribute) KYA (eval_framework.tasks.base.Language attribute) KYB (eval_framework.tasks.base.Language attribute) KYC (eval_framework.tasks.base.Language attribute) KYD (eval_framework.tasks.base.Language attribute) KYE (eval_framework.tasks.base.Language attribute) KYF (eval_framework.tasks.base.Language attribute) KYG (eval_framework.tasks.base.Language attribute) KYH (eval_framework.tasks.base.Language attribute) KYI (eval_framework.tasks.base.Language attribute) KYJ (eval_framework.tasks.base.Language attribute) KYK (eval_framework.tasks.base.Language attribute) KYL (eval_framework.tasks.base.Language attribute) KYM (eval_framework.tasks.base.Language attribute) KYN (eval_framework.tasks.base.Language attribute) KYO (eval_framework.tasks.base.Language attribute) KYP (eval_framework.tasks.base.Language attribute) KYQ (eval_framework.tasks.base.Language attribute) KYR (eval_framework.tasks.base.Language attribute) KYS (eval_framework.tasks.base.Language attribute) KYT (eval_framework.tasks.base.Language attribute) KYU (eval_framework.tasks.base.Language attribute) KYV (eval_framework.tasks.base.Language attribute) KYW (eval_framework.tasks.base.Language attribute) KYX (eval_framework.tasks.base.Language attribute) KYY (eval_framework.tasks.base.Language attribute) KYZ (eval_framework.tasks.base.Language attribute) KZA (eval_framework.tasks.base.Language attribute) KZB (eval_framework.tasks.base.Language attribute) KZC (eval_framework.tasks.base.Language attribute) KZD (eval_framework.tasks.base.Language attribute) KZE (eval_framework.tasks.base.Language attribute) KZF (eval_framework.tasks.base.Language attribute) KZG (eval_framework.tasks.base.Language attribute) KZH (eval_framework.tasks.base.Language attribute) KZI (eval_framework.tasks.base.Language attribute) KZJ (eval_framework.tasks.base.Language attribute) KZK (eval_framework.tasks.base.Language attribute) KZL (eval_framework.tasks.base.Language attribute) KZM (eval_framework.tasks.base.Language attribute) KZN (eval_framework.tasks.base.Language attribute) KZO (eval_framework.tasks.base.Language attribute) KZP (eval_framework.tasks.base.Language attribute) KZQ (eval_framework.tasks.base.Language attribute) KZR (eval_framework.tasks.base.Language attribute) KZS (eval_framework.tasks.base.Language attribute) KZT (eval_framework.tasks.base.Language attribute) KZU (eval_framework.tasks.base.Language attribute) KZV (eval_framework.tasks.base.Language attribute) KZW (eval_framework.tasks.base.Language attribute) KZX (eval_framework.tasks.base.Language attribute) KZY (eval_framework.tasks.base.Language attribute) KZZ (eval_framework.tasks.base.Language attribute) L LAA (eval_framework.tasks.base.Language attribute) LAB (eval_framework.tasks.base.Language attribute) LAC (eval_framework.tasks.base.Language attribute) LAD (eval_framework.tasks.base.Language attribute) LAE (eval_framework.tasks.base.Language attribute) LAF (eval_framework.tasks.base.Language attribute) LAG (eval_framework.tasks.base.Language attribute) LAH (eval_framework.tasks.base.Language attribute) LAI (eval_framework.tasks.base.Language attribute) LAJ (eval_framework.tasks.base.Language attribute) LAK (eval_framework.tasks.base.Language attribute) LAL (eval_framework.tasks.base.Language attribute) LAM (eval_framework.tasks.base.Language attribute) LAN (eval_framework.tasks.base.Language attribute) Language (class in eval_framework.tasks.base) language (eval_framework.metrics.llm.llm_judge_format_correctness.LLMJudgeFormatCorrectnessContext attribute) LANGUAGE (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalDe attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalFiSv attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATHReasoning attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.TRUTHFULQA_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.TRUTHFULQA_EU20_FR attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winox.WINOX_DE attribute) (eval_framework.tasks.benchmarks.winox.WINOX_FR attribute) (eval_framework.tasks.benchmarks.wmt.WMT14 attribute) (eval_framework.tasks.benchmarks.wmt.WMT14_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT16 attribute) (eval_framework.tasks.benchmarks.wmt.WMT16_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT20 attribute) (eval_framework.tasks.benchmarks.wmt.WMT20_INSTRUCT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) LANGUAGE_SHORT_CODE (eval_framework.tasks.benchmarks.winox.WINOX attribute) (eval_framework.tasks.benchmarks.winox.WINOX_DE attribute) (eval_framework.tasks.benchmarks.winox.WINOX_FR attribute) LanguageChecker (class in eval_framework.metrics.completion.language_checker) LanguageConsistencyChecker (class in eval_framework.metrics.completion.language_checker) LanguageRawConsistencyChecker (class in eval_framework.metrics.completion.language_checker) LAO (eval_framework.tasks.base.Language attribute) LAP (eval_framework.tasks.base.Language attribute) LAQ (eval_framework.tasks.base.Language attribute) LAR (eval_framework.tasks.base.Language attribute) LAS (eval_framework.tasks.base.Language attribute) LAT (eval_framework.tasks.base.Language attribute) LAU (eval_framework.tasks.base.Language attribute) LAV (eval_framework.tasks.base.Language attribute) LAW (eval_framework.tasks.base.Language attribute) LAX (eval_framework.tasks.base.Language attribute) LAY (eval_framework.tasks.base.Language attribute) LAZ (eval_framework.tasks.base.Language attribute) LBA (eval_framework.tasks.base.Language attribute) LBB (eval_framework.tasks.base.Language attribute) LBC (eval_framework.tasks.base.Language attribute) LBE (eval_framework.tasks.base.Language attribute) LBF (eval_framework.tasks.base.Language attribute) LBG (eval_framework.tasks.base.Language attribute) LBI (eval_framework.tasks.base.Language attribute) LBJ (eval_framework.tasks.base.Language attribute) LBK (eval_framework.tasks.base.Language attribute) LBL (eval_framework.tasks.base.Language attribute) LBM (eval_framework.tasks.base.Language attribute) LBN (eval_framework.tasks.base.Language attribute) LBO (eval_framework.tasks.base.Language attribute) LBQ (eval_framework.tasks.base.Language attribute) LBR (eval_framework.tasks.base.Language attribute) LBS (eval_framework.tasks.base.Language attribute) LBT (eval_framework.tasks.base.Language attribute) LBU (eval_framework.tasks.base.Language attribute) LBV (eval_framework.tasks.base.Language attribute) LBW (eval_framework.tasks.base.Language attribute) LBX (eval_framework.tasks.base.Language attribute) LBY (eval_framework.tasks.base.Language attribute) LBZ (eval_framework.tasks.base.Language attribute) LCC (eval_framework.tasks.base.Language attribute) LCD (eval_framework.tasks.base.Language attribute) LCE (eval_framework.tasks.base.Language attribute) LCF (eval_framework.tasks.base.Language attribute) LCH (eval_framework.tasks.base.Language attribute) LCL (eval_framework.tasks.base.Language attribute) LCM (eval_framework.tasks.base.Language attribute) LCP (eval_framework.tasks.base.Language attribute) LCQ (eval_framework.tasks.base.Language attribute) LCS (eval_framework.tasks.base.Language attribute) LDA (eval_framework.tasks.base.Language attribute) LDB (eval_framework.tasks.base.Language attribute) LDD (eval_framework.tasks.base.Language attribute) LDG (eval_framework.tasks.base.Language attribute) LDH (eval_framework.tasks.base.Language attribute) LDI (eval_framework.tasks.base.Language attribute) LDJ (eval_framework.tasks.base.Language attribute) LDK (eval_framework.tasks.base.Language attribute) LDL (eval_framework.tasks.base.Language attribute) LDM (eval_framework.tasks.base.Language attribute) LDN (eval_framework.tasks.base.Language attribute) LDO (eval_framework.tasks.base.Language attribute) LDP (eval_framework.tasks.base.Language attribute) LDQ (eval_framework.tasks.base.Language attribute) LEA (eval_framework.tasks.base.Language attribute) LEB (eval_framework.tasks.base.Language attribute) LEC (eval_framework.tasks.base.Language attribute) LED (eval_framework.tasks.base.Language attribute) LEE (eval_framework.tasks.base.Language attribute) LEF (eval_framework.tasks.base.Language attribute) LEG (eval_framework.tasks.base.Language attribute) LEH (eval_framework.tasks.base.Language attribute) LEI (eval_framework.tasks.base.Language attribute) LEJ (eval_framework.tasks.base.Language attribute) LEK (eval_framework.tasks.base.Language attribute) LEL (eval_framework.tasks.base.Language attribute) LEM (eval_framework.tasks.base.Language attribute) LEN (eval_framework.tasks.base.Language attribute) LengthControl (class in eval_framework.metrics.completion.length_control) LengthRequirementType (class in eval_framework.metrics.completion.length_control) LengthRequirementUnit (class in eval_framework.metrics.completion.length_control) LEO (eval_framework.tasks.base.Language attribute) LEP (eval_framework.tasks.base.Language attribute) LEQ (eval_framework.tasks.base.Language attribute) LER (eval_framework.tasks.base.Language attribute) LES (eval_framework.tasks.base.Language attribute) LET (eval_framework.tasks.base.Language attribute) LEU (eval_framework.tasks.base.Language attribute) LEV (eval_framework.tasks.base.Language attribute) LEW (eval_framework.tasks.base.Language attribute) LEX (eval_framework.tasks.base.Language attribute) LEY (eval_framework.tasks.base.Language attribute) LEZ (eval_framework.tasks.base.Language attribute) LFA (eval_framework.tasks.base.Language attribute) LFN (eval_framework.tasks.base.Language attribute) LGA (eval_framework.tasks.base.Language attribute) LGB (eval_framework.tasks.base.Language attribute) LGG (eval_framework.tasks.base.Language attribute) LGH (eval_framework.tasks.base.Language attribute) LGI (eval_framework.tasks.base.Language attribute) LGK (eval_framework.tasks.base.Language attribute) LGL (eval_framework.tasks.base.Language attribute) LGM (eval_framework.tasks.base.Language attribute) LGN (eval_framework.tasks.base.Language attribute) LGO (eval_framework.tasks.base.Language attribute) LGQ (eval_framework.tasks.base.Language attribute) LGR (eval_framework.tasks.base.Language attribute) LGS (eval_framework.tasks.base.Language attribute) LGT (eval_framework.tasks.base.Language attribute) LGU (eval_framework.tasks.base.Language attribute) LGZ (eval_framework.tasks.base.Language attribute) LHA (eval_framework.tasks.base.Language attribute) LHH (eval_framework.tasks.base.Language attribute) LHI (eval_framework.tasks.base.Language attribute) LHL (eval_framework.tasks.base.Language attribute) LHM (eval_framework.tasks.base.Language attribute) LHN (eval_framework.tasks.base.Language attribute) LHP (eval_framework.tasks.base.Language attribute) LHS (eval_framework.tasks.base.Language attribute) LHT (eval_framework.tasks.base.Language attribute) LHU (eval_framework.tasks.base.Language attribute) LIA (eval_framework.tasks.base.Language attribute) LIB (eval_framework.tasks.base.Language attribute) LIC (eval_framework.tasks.base.Language attribute) LID (eval_framework.tasks.base.Language attribute) LIE (eval_framework.tasks.base.Language attribute) LIF (eval_framework.tasks.base.Language attribute) LIG (eval_framework.tasks.base.Language attribute) LIH (eval_framework.tasks.base.Language attribute) LII (eval_framework.tasks.base.Language attribute) LIJ (eval_framework.tasks.base.Language attribute) LIK (eval_framework.tasks.base.Language attribute) LIL (eval_framework.tasks.base.Language attribute) LIM (eval_framework.tasks.base.Language attribute) LIN (eval_framework.tasks.base.Language attribute) LINEWISE_BLEU (class in eval_framework.metrics.completion.bleu) LINEWISE_CHRF (class in eval_framework.metrics.completion.chrf) LINEWISE_TER (class in eval_framework.metrics.completion.ter) LIO (eval_framework.tasks.base.Language attribute) LIP (eval_framework.tasks.base.Language attribute) LIQ (eval_framework.tasks.base.Language attribute) LIR (eval_framework.tasks.base.Language attribute) LIS (eval_framework.tasks.base.Language attribute) LIT (eval_framework.tasks.base.Language attribute) LIU (eval_framework.tasks.base.Language attribute) LIV (eval_framework.tasks.base.Language attribute) LIW (eval_framework.tasks.base.Language attribute) LIX (eval_framework.tasks.base.Language attribute) LIY (eval_framework.tasks.base.Language attribute) LIZ (eval_framework.tasks.base.Language attribute) LJA (eval_framework.tasks.base.Language attribute) LJE (eval_framework.tasks.base.Language attribute) LJI (eval_framework.tasks.base.Language attribute) LJL (eval_framework.tasks.base.Language attribute) LJP (eval_framework.tasks.base.Language attribute) LJW (eval_framework.tasks.base.Language attribute) LJX (eval_framework.tasks.base.Language attribute) LKA (eval_framework.tasks.base.Language attribute) LKB (eval_framework.tasks.base.Language attribute) LKC (eval_framework.tasks.base.Language attribute) LKD (eval_framework.tasks.base.Language attribute) LKE (eval_framework.tasks.base.Language attribute) LKH (eval_framework.tasks.base.Language attribute) LKI (eval_framework.tasks.base.Language attribute) LKJ (eval_framework.tasks.base.Language attribute) LKL (eval_framework.tasks.base.Language attribute) LKM (eval_framework.tasks.base.Language attribute) LKN (eval_framework.tasks.base.Language attribute) LKO (eval_framework.tasks.base.Language attribute) LKR (eval_framework.tasks.base.Language attribute) LKS (eval_framework.tasks.base.Language attribute) LKT (eval_framework.tasks.base.Language attribute) LKU (eval_framework.tasks.base.Language attribute) LKY (eval_framework.tasks.base.Language attribute) LLA (eval_framework.tasks.base.Language attribute) Llama31_8B_Instruct_API (class in eval_framework.llm.aleph_alpha) LLB (eval_framework.tasks.base.Language attribute) LLC (eval_framework.tasks.base.Language attribute) LLD (eval_framework.tasks.base.Language attribute) LLE (eval_framework.tasks.base.Language attribute) LLF (eval_framework.tasks.base.Language attribute) LLG (eval_framework.tasks.base.Language attribute) LLH (eval_framework.tasks.base.Language attribute) LLI (eval_framework.tasks.base.Language attribute) LLJ (eval_framework.tasks.base.Language attribute) LLK (eval_framework.tasks.base.Language attribute) LLL (eval_framework.tasks.base.Language attribute) LLM (eval_framework.tasks.base.Language attribute) llm_args (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) llm_class (eval_framework.tasks.eval_config.EvalConfig attribute) llm_judge_class (eval_framework.tasks.eval_config.EvalConfig attribute) llm_judge_prompt (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) llm_judge_response (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) llm_name (eval_framework.context.determined.Hyperparameters attribute) LLM_NAME (eval_framework.llm.aleph_alpha.AlephAlphaAPIModel attribute) (eval_framework.llm.aleph_alpha.Llama31_8B_Instruct_API attribute) (eval_framework.llm.huggingface.BaseHFLLM attribute) (eval_framework.llm.huggingface.Pythia410m attribute) (eval_framework.llm.huggingface.Qwen3_0_6B attribute) (eval_framework.llm.huggingface.SmolLM135M attribute) (eval_framework.llm.huggingface.Smollm135MInstruct attribute) (eval_framework.llm.openai.Deepseek_chat attribute) (eval_framework.llm.openai.Deepseek_chat_with_formatter attribute) (eval_framework.llm.openai.Deepseek_reasoner attribute) (eval_framework.llm.openai.OpenAI_davinci_002 attribute) (eval_framework.llm.openai.OpenAI_gpt_4o_mini attribute) (eval_framework.llm.openai.OpenAI_gpt_4o_mini_with_ConcatFormatter attribute) (eval_framework.llm.openai.OpenAIModel attribute) (eval_framework.llm.vllm.BaseVLLMModel attribute) (eval_framework.llm.vllm.Qwen3_0_6B_VLLM attribute) (eval_framework.llm.vllm.Qwen3_0_6B_VLLM_No_Thinking attribute) llm_name (eval_framework.result_processors.base.Result attribute) LLMJudgeAvoidsNames (class in eval_framework.metrics.llm.llm_judge_contains_names) LLMJudgeChatbotStyle (class in eval_framework.metrics.llm.llm_judge_chatbot_style) LLMJudgeCoherence (class in eval_framework.metrics.llm.llm_judge_coherence) LLMJudgeCompletionAccuracy (class in eval_framework.metrics.llm.llm_judge_completion_accuracy) LLMJudgeConciseness (class in eval_framework.metrics.llm.llm_judge_conciseness) LLMJudgeFormatCorrectness (class in eval_framework.metrics.llm.llm_judge_format_correctness) LLMJudgeFormatCorrectnessContext (class in eval_framework.metrics.llm.llm_judge_format_correctness) LLMJudgeInstruction (class in eval_framework.metrics.llm.llm_judge_instruction) LLMJudgeRefusal (class in eval_framework.metrics.llm.llm_judge_refusal) LLMJudgeSql (class in eval_framework.metrics.llm.llm_judge_sql) LLMJudgeSqlMetricContext (class in eval_framework.metrics.llm.llm_judge_sql) LLMJudgeWorldKnowledge (class in eval_framework.metrics.llm.llm_judge_world_knowledge) LLN (eval_framework.tasks.base.Language attribute) LLO (eval_framework.tasks.base.Language attribute) LLP (eval_framework.tasks.base.Language attribute) LLQ (eval_framework.tasks.base.Language attribute) LLS (eval_framework.tasks.base.Language attribute) LLU (eval_framework.tasks.base.Language attribute) LLX (eval_framework.tasks.base.Language attribute) LMA (eval_framework.tasks.base.Language attribute) LMB (eval_framework.tasks.base.Language attribute) LMC (eval_framework.tasks.base.Language attribute) LMD (eval_framework.tasks.base.Language attribute) LME (eval_framework.tasks.base.Language attribute) LMF (eval_framework.tasks.base.Language attribute) LMG (eval_framework.tasks.base.Language attribute) LMH (eval_framework.tasks.base.Language attribute) LMI (eval_framework.tasks.base.Language attribute) LMJ (eval_framework.tasks.base.Language attribute) LMK (eval_framework.tasks.base.Language attribute) LML (eval_framework.tasks.base.Language attribute) LMM (eval_framework.tasks.base.Language attribute) LMN (eval_framework.tasks.base.Language attribute) LMO (eval_framework.tasks.base.Language attribute) LMP (eval_framework.tasks.base.Language attribute) LMQ (eval_framework.tasks.base.Language attribute) LMR (eval_framework.tasks.base.Language attribute) LMS (eval_framework.tasks.base.Language attribute) LMT (eval_framework.tasks.base.Language attribute) LMU (eval_framework.tasks.base.Language attribute) LMV (eval_framework.tasks.base.Language attribute) LMW (eval_framework.tasks.base.Language attribute) LMX (eval_framework.tasks.base.Language attribute) LMY (eval_framework.tasks.base.Language attribute) LMZ (eval_framework.tasks.base.Language attribute) LNA (eval_framework.tasks.base.Language attribute) LNB (eval_framework.tasks.base.Language attribute) LNC (eval_framework.tasks.base.Language attribute) LND (eval_framework.tasks.base.Language attribute) LNG (eval_framework.tasks.base.Language attribute) LNH (eval_framework.tasks.base.Language attribute) LNI (eval_framework.tasks.base.Language attribute) LNJ (eval_framework.tasks.base.Language attribute) LNL (eval_framework.tasks.base.Language attribute) LNM (eval_framework.tasks.base.Language attribute) LNN (eval_framework.tasks.base.Language attribute) LNO (eval_framework.tasks.base.Language attribute) LNS (eval_framework.tasks.base.Language attribute) LNT (eval_framework.tasks.base.Language attribute) LNU (eval_framework.tasks.base.Language attribute) LNW (eval_framework.tasks.base.Language attribute) LNZ (eval_framework.tasks.base.Language attribute) LOA (eval_framework.tasks.base.Language attribute) load_extra_tasks() (in module eval_framework.tasks.task_loader) load_metadata() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) load_metrics_results() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) load_responses() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) LOB (eval_framework.tasks.base.Language attribute) LOC (eval_framework.tasks.base.Language attribute) LocalContext (class in eval_framework.context.local) LOD (eval_framework.tasks.base.Language attribute) LOE (eval_framework.tasks.base.Language attribute) LOF (eval_framework.tasks.base.Language attribute) LOG (eval_framework.tasks.base.Language attribute) LogicError LOGLIKELIHOODS (eval_framework.tasks.base.ResponseType attribute) logprobs() (eval_framework.llm.aleph_alpha.AlephAlphaAPIModel method) (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.huggingface.BaseHFLLM method) (eval_framework.llm.openai.OpenAIEmbeddingModel method) (eval_framework.llm.openai.OpenAIModel method) (eval_framework.llm.vllm.BaseVLLMModel method) LOH (eval_framework.tasks.base.Language attribute) LOI (eval_framework.tasks.base.Language attribute) LOJ (eval_framework.tasks.base.Language attribute) LOK (eval_framework.tasks.base.Language attribute) LOL (eval_framework.tasks.base.Language attribute) LOM (eval_framework.tasks.base.Language attribute) LON (eval_framework.tasks.base.Language attribute) LOO (eval_framework.tasks.base.Language attribute) LOP (eval_framework.tasks.base.Language attribute) LOQ (eval_framework.tasks.base.Language attribute) LOR (eval_framework.tasks.base.Language attribute) LOS (eval_framework.tasks.base.Language attribute) LOT (eval_framework.tasks.base.Language attribute) LOU (eval_framework.tasks.base.Language attribute) LOV (eval_framework.tasks.base.Language attribute) LOW (eval_framework.tasks.base.Language attribute) LOX (eval_framework.tasks.base.Language attribute) LOY (eval_framework.tasks.base.Language attribute) LOZ (eval_framework.tasks.base.Language attribute) LPA (eval_framework.tasks.base.Language attribute) LPE (eval_framework.tasks.base.Language attribute) LPN (eval_framework.tasks.base.Language attribute) LPO (eval_framework.tasks.base.Language attribute) LPX (eval_framework.tasks.base.Language attribute) LQR (eval_framework.tasks.base.Language attribute) LRA (eval_framework.tasks.base.Language attribute) LRC (eval_framework.tasks.base.Language attribute) LRE (eval_framework.tasks.base.Language attribute) LRG (eval_framework.tasks.base.Language attribute) LRI (eval_framework.tasks.base.Language attribute) LRK (eval_framework.tasks.base.Language attribute) LRL (eval_framework.tasks.base.Language attribute) LRM (eval_framework.tasks.base.Language attribute) LRN (eval_framework.tasks.base.Language attribute) LRO (eval_framework.tasks.base.Language attribute) LRR (eval_framework.tasks.base.Language attribute) LRT (eval_framework.tasks.base.Language attribute) LRV (eval_framework.tasks.base.Language attribute) LRZ (eval_framework.tasks.base.Language attribute) LSA (eval_framework.tasks.base.Language attribute) LSB (eval_framework.tasks.base.Language attribute) LSC (eval_framework.tasks.base.Language attribute) LSD (eval_framework.tasks.base.Language attribute) LSE (eval_framework.tasks.base.Language attribute) LSG (eval_framework.tasks.base.Language attribute) LSH (eval_framework.tasks.base.Language attribute) LSI (eval_framework.tasks.base.Language attribute) LSL (eval_framework.tasks.base.Language attribute) LSM (eval_framework.tasks.base.Language attribute) LSN (eval_framework.tasks.base.Language attribute) LSO (eval_framework.tasks.base.Language attribute) LSP (eval_framework.tasks.base.Language attribute) LSR (eval_framework.tasks.base.Language attribute) LSS (eval_framework.tasks.base.Language attribute) LST (eval_framework.tasks.base.Language attribute) LSV (eval_framework.tasks.base.Language attribute) LSW (eval_framework.tasks.base.Language attribute) LSY (eval_framework.tasks.base.Language attribute) LTC (eval_framework.tasks.base.Language attribute) LTG (eval_framework.tasks.base.Language attribute) LTH (eval_framework.tasks.base.Language attribute) LTI (eval_framework.tasks.base.Language attribute) LTN (eval_framework.tasks.base.Language attribute) LTO (eval_framework.tasks.base.Language attribute) LTS (eval_framework.tasks.base.Language attribute) LTU (eval_framework.tasks.base.Language attribute) LTZ (eval_framework.tasks.base.Language attribute) LUA (eval_framework.tasks.base.Language attribute) LUB (eval_framework.tasks.base.Language attribute) LUC (eval_framework.tasks.base.Language attribute) LUD (eval_framework.tasks.base.Language attribute) LUE (eval_framework.tasks.base.Language attribute) LUF (eval_framework.tasks.base.Language attribute) LUG (eval_framework.tasks.base.Language attribute) LUH (eval_framework.tasks.base.Language attribute) LUI (eval_framework.tasks.base.Language attribute) LUJ (eval_framework.tasks.base.Language attribute) LUK (eval_framework.tasks.base.Language attribute) LUL (eval_framework.tasks.base.Language attribute) LUM (eval_framework.tasks.base.Language attribute) LUN (eval_framework.tasks.base.Language attribute) LUO (eval_framework.tasks.base.Language attribute) LUP (eval_framework.tasks.base.Language attribute) LUQ (eval_framework.tasks.base.Language attribute) LUR (eval_framework.tasks.base.Language attribute) LUS (eval_framework.tasks.base.Language attribute) LUT (eval_framework.tasks.base.Language attribute) LUU (eval_framework.tasks.base.Language attribute) LUV (eval_framework.tasks.base.Language attribute) LUW (eval_framework.tasks.base.Language attribute) LUY (eval_framework.tasks.base.Language attribute) LUZ (eval_framework.tasks.base.Language attribute) LVA (eval_framework.tasks.base.Language attribute) LVI (eval_framework.tasks.base.Language attribute) LVK (eval_framework.tasks.base.Language attribute) LVL (eval_framework.tasks.base.Language attribute) LVS (eval_framework.tasks.base.Language attribute) LVU (eval_framework.tasks.base.Language attribute) LWA (eval_framework.tasks.base.Language attribute) LWE (eval_framework.tasks.base.Language attribute) LWG (eval_framework.tasks.base.Language attribute) LWH (eval_framework.tasks.base.Language attribute) LWL (eval_framework.tasks.base.Language attribute) LWM (eval_framework.tasks.base.Language attribute) LWO (eval_framework.tasks.base.Language attribute) LWS (eval_framework.tasks.base.Language attribute) LWT (eval_framework.tasks.base.Language attribute) LWU (eval_framework.tasks.base.Language attribute) LWW (eval_framework.tasks.base.Language attribute) LXM (eval_framework.tasks.base.Language attribute) LYA (eval_framework.tasks.base.Language attribute) LYG (eval_framework.tasks.base.Language attribute) LYN (eval_framework.tasks.base.Language attribute) LZH (eval_framework.tasks.base.Language attribute) LZL (eval_framework.tasks.base.Language attribute) LZN (eval_framework.tasks.base.Language attribute) LZZ (eval_framework.tasks.base.Language attribute) M MAA (eval_framework.tasks.base.Language attribute) MAB (eval_framework.tasks.base.Language attribute) MAD (eval_framework.tasks.base.Language attribute) MAE (eval_framework.tasks.base.Language attribute) MAF (eval_framework.tasks.base.Language attribute) MAG (eval_framework.tasks.base.Language attribute) MAH (eval_framework.tasks.base.Language attribute) MAI (eval_framework.tasks.base.Language attribute) main() (in module eval_framework.main) MAJ (eval_framework.tasks.base.Language attribute) MAK (eval_framework.tasks.base.Language attribute) make_sure_all_hf_datasets_are_in_cache() (in module eval_framework.tasks.task_names) MAL (eval_framework.tasks.base.Language attribute) MAM (eval_framework.tasks.base.Language attribute) MAN (eval_framework.tasks.base.Language attribute) map_language_to_value() (in module eval_framework.response_generator) MAQ (eval_framework.tasks.base.Language attribute) MAR (eval_framework.tasks.base.Language attribute) MAS (eval_framework.tasks.base.Language attribute) MAT (eval_framework.tasks.base.Language attribute) matches_column_count (eval_framework.metrics.llm.llm_judge_sql.SqlOutputComparison attribute) matches_results_count (eval_framework.metrics.llm.llm_judge_sql.SqlOutputComparison attribute) MATH (class in eval_framework.tasks.benchmarks.math_reasoning) MATH500 (class in eval_framework.tasks.benchmarks.math_reasoning) MATHLvl5 (class in eval_framework.tasks.benchmarks.math_reasoning) MATHReasoning (class in eval_framework.tasks.benchmarks.math_reasoning) MathReasoningCompletion (class in eval_framework.metrics.completion.math_reasoning_completion) MAU (eval_framework.tasks.base.Language attribute) MAV (eval_framework.tasks.base.Language attribute) MAW (eval_framework.tasks.base.Language attribute) MAX (eval_framework.metrics.completion.length_control.LengthRequirementType attribute) (eval_framework.tasks.base.Language attribute) max_seq_length (eval_framework.llm.vllm.BaseVLLMModel property) max_tokens (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) MAZ (eval_framework.tasks.base.Language attribute) MBA (eval_framework.tasks.base.Language attribute) MBB (eval_framework.tasks.base.Language attribute) MBC (eval_framework.tasks.base.Language attribute) MBD (eval_framework.tasks.base.Language attribute) MBE (eval_framework.tasks.base.Language attribute) MBF (eval_framework.tasks.base.Language attribute) MBG (eval_framework.tasks.base.Language attribute) MBH (eval_framework.tasks.base.Language attribute) MBI (eval_framework.tasks.base.Language attribute) MBJ (eval_framework.tasks.base.Language attribute) MBK (eval_framework.tasks.base.Language attribute) MBL (eval_framework.tasks.base.Language attribute) MBM (eval_framework.tasks.base.Language attribute) MBN (eval_framework.tasks.base.Language attribute) MBO (eval_framework.tasks.base.Language attribute) MBP (eval_framework.tasks.base.Language attribute) MBPP (class in eval_framework.tasks.benchmarks.mbpp) MBPP_PROMPT_WITHOUT_TESTS (class in eval_framework.tasks.benchmarks.mbpp) MBPP_PROMPT_WITHOUT_TESTS_SANITIZED (class in eval_framework.tasks.benchmarks.mbpp) MBPP_SANITIZED (class in eval_framework.tasks.benchmarks.mbpp) MBPPMetricContext (class in eval_framework.tasks.benchmarks.mbpp) MBQ (eval_framework.tasks.base.Language attribute) MBR (eval_framework.tasks.base.Language attribute) MBS (eval_framework.tasks.base.Language attribute) MBT (eval_framework.tasks.base.Language attribute) MBU (eval_framework.tasks.base.Language attribute) MBV (eval_framework.tasks.base.Language attribute) MBW (eval_framework.tasks.base.Language attribute) MBX (eval_framework.tasks.base.Language attribute) MBY (eval_framework.tasks.base.Language attribute) MBZ (eval_framework.tasks.base.Language attribute) MCA (eval_framework.tasks.base.Language attribute) MCB (eval_framework.tasks.base.Language attribute) MCC (eval_framework.tasks.base.Language attribute) MCD (eval_framework.tasks.base.Language attribute) MCE (eval_framework.tasks.base.Language attribute) MCF (eval_framework.tasks.base.Language attribute) MCG (eval_framework.tasks.base.Language attribute) MCH (eval_framework.tasks.base.Language attribute) MCI (eval_framework.tasks.base.Language attribute) MCJ (eval_framework.tasks.base.Language attribute) MCK (eval_framework.tasks.base.Language attribute) MCL (eval_framework.tasks.base.Language attribute) MCM (eval_framework.tasks.base.Language attribute) MCN (eval_framework.tasks.base.Language attribute) MCO (eval_framework.tasks.base.Language attribute) MCP (eval_framework.tasks.base.Language attribute) MCQ (eval_framework.tasks.base.Language attribute) MCR (eval_framework.tasks.base.Language attribute) MCS (eval_framework.tasks.base.Language attribute) MCT (eval_framework.tasks.base.Language attribute) MCU (eval_framework.tasks.base.Language attribute) MCV (eval_framework.tasks.base.Language attribute) MCW (eval_framework.tasks.base.Language attribute) MCX (eval_framework.tasks.base.Language attribute) MCY (eval_framework.tasks.base.Language attribute) MCZ (eval_framework.tasks.base.Language attribute) MDA (eval_framework.tasks.base.Language attribute) MDB (eval_framework.tasks.base.Language attribute) MDC (eval_framework.tasks.base.Language attribute) MDD (eval_framework.tasks.base.Language attribute) MDE (eval_framework.tasks.base.Language attribute) MDF (eval_framework.tasks.base.Language attribute) MDG (eval_framework.tasks.base.Language attribute) MDH (eval_framework.tasks.base.Language attribute) MDI (eval_framework.tasks.base.Language attribute) MDJ (eval_framework.tasks.base.Language attribute) MDK (eval_framework.tasks.base.Language attribute) MDL (eval_framework.tasks.base.Language attribute) MDM (eval_framework.tasks.base.Language attribute) MDN (eval_framework.tasks.base.Language attribute) MDO (eval_framework.tasks.base.Language attribute) MDP (eval_framework.tasks.base.Language attribute) MDQ (eval_framework.tasks.base.Language attribute) MDR (eval_framework.tasks.base.Language attribute) MDS (eval_framework.tasks.base.Language attribute) MDT (eval_framework.tasks.base.Language attribute) MDU (eval_framework.tasks.base.Language attribute) MDV (eval_framework.tasks.base.Language attribute) MDW (eval_framework.tasks.base.Language attribute) MDX (eval_framework.tasks.base.Language attribute) MDY (eval_framework.tasks.base.Language attribute) MDZ (eval_framework.tasks.base.Language attribute) MEA (eval_framework.tasks.base.Language attribute) MEB (eval_framework.tasks.base.Language attribute) MEC (eval_framework.tasks.base.Language attribute) MED (eval_framework.tasks.base.Language attribute) MEE (eval_framework.tasks.base.Language attribute) MEF (eval_framework.tasks.base.Language attribute) MEG (eval_framework.tasks.base.Language attribute) MEH (eval_framework.tasks.base.Language attribute) MEI (eval_framework.tasks.base.Language attribute) MEJ (eval_framework.tasks.base.Language attribute) MEK (eval_framework.tasks.base.Language attribute) MEL (eval_framework.tasks.base.Language attribute) MEM (eval_framework.tasks.base.Language attribute) MEN (eval_framework.tasks.base.Language attribute) MEO (eval_framework.tasks.base.Language attribute) MEP (eval_framework.tasks.base.Language attribute) MEQ (eval_framework.tasks.base.Language attribute) MER (eval_framework.tasks.base.Language attribute) MES (eval_framework.tasks.base.Language attribute) messages (eval_framework.tasks.base.Sample attribute) MET (eval_framework.tasks.base.Language attribute) metric_class_name (eval_framework.result_processors.base.Result attribute) metric_name (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) MetricResult (class in eval_framework.metrics.base) METRICS (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc.ARC_IDK attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.copa.COPA_IDK attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA_COT attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA_IDK attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG_IDK attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBenchCompletion attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBenchLoglikelihood attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATHReasoning attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_IDK attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA_IDK attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.piqa.PIQA_IDK attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ_IDK attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness_IDK attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.RenderableStructEval attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA_IDK attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER_IDK attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE_IDK attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_GOV_REPORT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_MUSIQUE attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_NARRATIVEQA attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QASPER attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QMSUM attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SPACE_DIGEST attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SQUALITY attribute) MEU (eval_framework.tasks.base.Language attribute) MEV (eval_framework.tasks.base.Language attribute) MEW (eval_framework.tasks.base.Language attribute) MEY (eval_framework.tasks.base.Language attribute) MEZ (eval_framework.tasks.base.Language attribute) MFA (eval_framework.tasks.base.Language attribute) MFB (eval_framework.tasks.base.Language attribute) MFC (eval_framework.tasks.base.Language attribute) MFD (eval_framework.tasks.base.Language attribute) MFE (eval_framework.tasks.base.Language attribute) MFF (eval_framework.tasks.base.Language attribute) MFG (eval_framework.tasks.base.Language attribute) MFH (eval_framework.tasks.base.Language attribute) MFI (eval_framework.tasks.base.Language attribute) MFJ (eval_framework.tasks.base.Language attribute) MFK (eval_framework.tasks.base.Language attribute) MFL (eval_framework.tasks.base.Language attribute) MFM (eval_framework.tasks.base.Language attribute) MFN (eval_framework.tasks.base.Language attribute) MFO (eval_framework.tasks.base.Language attribute) MFP (eval_framework.tasks.base.Language attribute) MFQ (eval_framework.tasks.base.Language attribute) MFR (eval_framework.tasks.base.Language attribute) MFS (eval_framework.tasks.base.Language attribute) MFT (eval_framework.tasks.base.Language attribute) MFU (eval_framework.tasks.base.Language attribute) MFV (eval_framework.tasks.base.Language attribute) MFW (eval_framework.tasks.base.Language attribute) MFX (eval_framework.tasks.base.Language attribute) MFY (eval_framework.tasks.base.Language attribute) MFZ (eval_framework.tasks.base.Language attribute) MGA (eval_framework.tasks.base.Language attribute) MGB (eval_framework.tasks.base.Language attribute) MGC (eval_framework.tasks.base.Language attribute) MGD (eval_framework.tasks.base.Language attribute) MGE (eval_framework.tasks.base.Language attribute) MGF (eval_framework.tasks.base.Language attribute) MGG (eval_framework.tasks.base.Language attribute) MGH (eval_framework.tasks.base.Language attribute) MGI (eval_framework.tasks.base.Language attribute) MGJ (eval_framework.tasks.base.Language attribute) MGK (eval_framework.tasks.base.Language attribute) MGL (eval_framework.tasks.base.Language attribute) MGM (eval_framework.tasks.base.Language attribute) MGN (eval_framework.tasks.base.Language attribute) MGO (eval_framework.tasks.base.Language attribute) MGP (eval_framework.tasks.base.Language attribute) MGQ (eval_framework.tasks.base.Language attribute) MGR (eval_framework.tasks.base.Language attribute) MGS (eval_framework.tasks.base.Language attribute) MGT (eval_framework.tasks.base.Language attribute) MGU (eval_framework.tasks.base.Language attribute) MGV (eval_framework.tasks.base.Language attribute) MGW (eval_framework.tasks.base.Language attribute) MGX (eval_framework.tasks.base.Language attribute) MGY (eval_framework.tasks.base.Language attribute) MGZ (eval_framework.tasks.base.Language attribute) MHA (eval_framework.tasks.base.Language attribute) MHB (eval_framework.tasks.base.Language attribute) MHC (eval_framework.tasks.base.Language attribute) MHD (eval_framework.tasks.base.Language attribute) MHE (eval_framework.tasks.base.Language attribute) MHF (eval_framework.tasks.base.Language attribute) MHG (eval_framework.tasks.base.Language attribute) MHH (eval_framework.tasks.base.Language attribute) MHI (eval_framework.tasks.base.Language attribute) MHJ (eval_framework.tasks.base.Language attribute) MHK (eval_framework.tasks.base.Language attribute) MHL (eval_framework.tasks.base.Language attribute) MHM (eval_framework.tasks.base.Language attribute) MHN (eval_framework.tasks.base.Language attribute) MHO (eval_framework.tasks.base.Language attribute) MHP (eval_framework.tasks.base.Language attribute) MHQ (eval_framework.tasks.base.Language attribute) MHR (eval_framework.tasks.base.Language attribute) MHS (eval_framework.tasks.base.Language attribute) MHT (eval_framework.tasks.base.Language attribute) MHU (eval_framework.tasks.base.Language attribute) MHV (eval_framework.tasks.base.Language attribute) MHW (eval_framework.tasks.base.Language attribute) MHX (eval_framework.tasks.base.Language attribute) MHY (eval_framework.tasks.base.Language attribute) MHZ (eval_framework.tasks.base.Language attribute) MIA (eval_framework.tasks.base.Language attribute) MIB (eval_framework.tasks.base.Language attribute) MIC (eval_framework.tasks.base.Language attribute) MID (eval_framework.tasks.base.Language attribute) MIE (eval_framework.tasks.base.Language attribute) MIF (eval_framework.tasks.base.Language attribute) MIG (eval_framework.tasks.base.Language attribute) MIH (eval_framework.tasks.base.Language attribute) MII (eval_framework.tasks.base.Language attribute) MIJ (eval_framework.tasks.base.Language attribute) MIK (eval_framework.tasks.base.Language attribute) MIL (eval_framework.tasks.base.Language attribute) MIM (eval_framework.tasks.base.Language attribute) MIN (eval_framework.metrics.completion.length_control.LengthRequirementType attribute) (eval_framework.tasks.base.Language attribute) MIO (eval_framework.tasks.base.Language attribute) MIP (eval_framework.tasks.base.Language attribute) MIQ (eval_framework.tasks.base.Language attribute) MIR (eval_framework.tasks.base.Language attribute) MIS (eval_framework.tasks.base.Language attribute) MistralAdapter (class in eval_framework.llm.mistral) MistralVLLM (class in eval_framework.llm.mistral) MIT (eval_framework.tasks.base.Language attribute) MIU (eval_framework.tasks.base.Language attribute) MIV (eval_framework.tasks.base.Language attribute) MIW (eval_framework.tasks.base.Language attribute) MIX (eval_framework.tasks.base.Language attribute) MIY (eval_framework.tasks.base.Language attribute) MIZ (eval_framework.tasks.base.Language attribute) MJA (eval_framework.tasks.base.Language attribute) MJB (eval_framework.tasks.base.Language attribute) MJC (eval_framework.tasks.base.Language attribute) MJD (eval_framework.tasks.base.Language attribute) MJE (eval_framework.tasks.base.Language attribute) MJG (eval_framework.tasks.base.Language attribute) MJH (eval_framework.tasks.base.Language attribute) MJI (eval_framework.tasks.base.Language attribute) MJJ (eval_framework.tasks.base.Language attribute) MJK (eval_framework.tasks.base.Language attribute) MJL (eval_framework.tasks.base.Language attribute) MJM (eval_framework.tasks.base.Language attribute) MJN (eval_framework.tasks.base.Language attribute) MJO (eval_framework.tasks.base.Language attribute) MJP (eval_framework.tasks.base.Language attribute) MJQ (eval_framework.tasks.base.Language attribute) MJR (eval_framework.tasks.base.Language attribute) MJS (eval_framework.tasks.base.Language attribute) MJT (eval_framework.tasks.base.Language attribute) MJU (eval_framework.tasks.base.Language attribute) MJV (eval_framework.tasks.base.Language attribute) MJW (eval_framework.tasks.base.Language attribute) MJX (eval_framework.tasks.base.Language attribute) MJY (eval_framework.tasks.base.Language attribute) MJZ (eval_framework.tasks.base.Language attribute) MKA (eval_framework.tasks.base.Language attribute) MKB (eval_framework.tasks.base.Language attribute) MKC (eval_framework.tasks.base.Language attribute) MKD (eval_framework.tasks.base.Language attribute) MKE (eval_framework.tasks.base.Language attribute) MKF (eval_framework.tasks.base.Language attribute) MKG (eval_framework.tasks.base.Language attribute) MKI (eval_framework.tasks.base.Language attribute) MKJ (eval_framework.tasks.base.Language attribute) MKK (eval_framework.tasks.base.Language attribute) MKL (eval_framework.tasks.base.Language attribute) MKM (eval_framework.tasks.base.Language attribute) MKN (eval_framework.tasks.base.Language attribute) MKO (eval_framework.tasks.base.Language attribute) MKP (eval_framework.tasks.base.Language attribute) MKQ (eval_framework.tasks.base.Language attribute) MKR (eval_framework.tasks.base.Language attribute) MKS (eval_framework.tasks.base.Language attribute) MKT (eval_framework.tasks.base.Language attribute) MKU (eval_framework.tasks.base.Language attribute) MKV (eval_framework.tasks.base.Language attribute) MKW (eval_framework.tasks.base.Language attribute) MKX (eval_framework.tasks.base.Language attribute) MKY (eval_framework.tasks.base.Language attribute) MKZ (eval_framework.tasks.base.Language attribute) MLA (eval_framework.tasks.base.Language attribute) MLB (eval_framework.tasks.base.Language attribute) MLC (eval_framework.tasks.base.Language attribute) MLD (eval_framework.tasks.base.Language attribute) MLE (eval_framework.tasks.base.Language attribute) MLF (eval_framework.tasks.base.Language attribute) MLG (eval_framework.tasks.base.Language attribute) MLH (eval_framework.tasks.base.Language attribute) MLI (eval_framework.tasks.base.Language attribute) MLJ (eval_framework.tasks.base.Language attribute) MLK (eval_framework.tasks.base.Language attribute) MLL (eval_framework.tasks.base.Language attribute) MLM (eval_framework.tasks.base.Language attribute) MLN (eval_framework.tasks.base.Language attribute) MLO (eval_framework.tasks.base.Language attribute) MLP (eval_framework.tasks.base.Language attribute) MLQ (eval_framework.tasks.base.Language attribute) MLR (eval_framework.tasks.base.Language attribute) MLS (eval_framework.tasks.base.Language attribute) MLT (eval_framework.tasks.base.Language attribute) MLU (eval_framework.tasks.base.Language attribute) MLV (eval_framework.tasks.base.Language attribute) MLW (eval_framework.tasks.base.Language attribute) MLX (eval_framework.tasks.base.Language attribute) MLY (eval_framework.tasks.base.Language attribute) MLZ (eval_framework.tasks.base.Language attribute) MMA (eval_framework.tasks.base.Language attribute) MMB (eval_framework.tasks.base.Language attribute) MMC (eval_framework.tasks.base.Language attribute) MMD (eval_framework.tasks.base.Language attribute) MME (eval_framework.tasks.base.Language attribute) MMF (eval_framework.tasks.base.Language attribute) MMG (eval_framework.tasks.base.Language attribute) MMH (eval_framework.tasks.base.Language attribute) MMI (eval_framework.tasks.base.Language attribute) MMJ (eval_framework.tasks.base.Language attribute) MMK (eval_framework.tasks.base.Language attribute) MML (eval_framework.tasks.base.Language attribute) MMLU (class in eval_framework.tasks.benchmarks.mmlu) MMLU_COT (class in eval_framework.tasks.benchmarks.mmlu) MMLU_DE (class in eval_framework.tasks.benchmarks.mmlu_de) MMLU_EU20_DE (class in eval_framework.tasks.benchmarks.opengptx_eu20) MMLU_EU20_FR (class in eval_framework.tasks.benchmarks.opengptx_eu20) MMLU_IDK (class in eval_framework.tasks.benchmarks.mmlu) MMLU_PRO (class in eval_framework.tasks.benchmarks.mmlu_pro) MMLU_PRO_COT (class in eval_framework.tasks.benchmarks.mmlu_pro) MMLU_PRO_IDK (class in eval_framework.tasks.benchmarks.mmlu_pro) MMM (eval_framework.tasks.base.Language attribute) MMMLU (class in eval_framework.tasks.benchmarks.mmmlu) MMMLU_GERMAN_COT (class in eval_framework.tasks.benchmarks.mmmlu) MMN (eval_framework.tasks.base.Language attribute) MMO (eval_framework.tasks.base.Language attribute) MMP (eval_framework.tasks.base.Language attribute) MMQ (eval_framework.tasks.base.Language attribute) MMR (eval_framework.tasks.base.Language attribute) MMS (eval_framework.tasks.base.Language attribute) MMT (eval_framework.tasks.base.Language attribute) MMU (eval_framework.tasks.base.Language attribute) MMV (eval_framework.tasks.base.Language attribute) MMW (eval_framework.tasks.base.Language attribute) MMX (eval_framework.tasks.base.Language attribute) MMY (eval_framework.tasks.base.Language attribute) MMZ (eval_framework.tasks.base.Language attribute) MNA (eval_framework.tasks.base.Language attribute) MNB (eval_framework.tasks.base.Language attribute) MNC (eval_framework.tasks.base.Language attribute) MND (eval_framework.tasks.base.Language attribute) MNE (eval_framework.tasks.base.Language attribute) MNF (eval_framework.tasks.base.Language attribute) MNG (eval_framework.tasks.base.Language attribute) MNH (eval_framework.tasks.base.Language attribute) MNI (eval_framework.tasks.base.Language attribute) MNJ (eval_framework.tasks.base.Language attribute) MNK (eval_framework.tasks.base.Language attribute) MNL (eval_framework.tasks.base.Language attribute) MNM (eval_framework.tasks.base.Language attribute) MNN (eval_framework.tasks.base.Language attribute) MNP (eval_framework.tasks.base.Language attribute) MNQ (eval_framework.tasks.base.Language attribute) MNR (eval_framework.tasks.base.Language attribute) MNS (eval_framework.tasks.base.Language attribute) MNT (eval_framework.tasks.base.Language attribute) MNU (eval_framework.tasks.base.Language attribute) MNV (eval_framework.tasks.base.Language attribute) MNW (eval_framework.tasks.base.Language attribute) MNX (eval_framework.tasks.base.Language attribute) MNY (eval_framework.tasks.base.Language attribute) MNZ (eval_framework.tasks.base.Language attribute) MOA (eval_framework.tasks.base.Language attribute) MOB (eval_framework.tasks.base.Language attribute) MOC (eval_framework.tasks.base.Language attribute) MOD (eval_framework.tasks.base.Language attribute) model_config (eval_framework.base_config.BaseConfig attribute) (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.context.determined.TaskArgs attribute) (eval_framework.metrics.base.MetricResult attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext attribute) (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) (eval_framework.metrics.completion.placeholder_checker.PlaceholderCheckerMetricContext attribute) (eval_framework.metrics.completion.struct_eval_metrics.RenderableStructMetricContext attribute) (eval_framework.metrics.completion.struct_eval_metrics.StructMetricContext attribute) (eval_framework.metrics.completion.text_counter.ParagraphCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.SentenceCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.WordCounterMetricContext attribute) (eval_framework.metrics.llm.llm_judge_format_correctness.LLMJudgeFormatCorrectnessContext attribute) (eval_framework.metrics.llm.llm_judge_mtbench_pair.MTBenchJudgePairMetricContext attribute) (eval_framework.metrics.llm.llm_judge_mtbench_pair.PromptToJudge attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.MTBenchJudgeSingleMetricContext attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.PromptToJudge attribute) (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSqlMetricContext attribute) (eval_framework.metrics.llm.llm_judge_sql.SqlOutputComparison attribute) (eval_framework.metrics.llm.llm_judge_sql.SqlValidationResult attribute) (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.base.Sample attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEvalMetricContext attribute) (eval_framework.tasks.benchmarks.mbpp.MBPPMetricContext attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) (eval_framework.tasks.perturbation.PerturbationConfig attribute) model_json_dump() (eval_framework.tasks.eval_config.EvalConfig method) model_json_robust_subset_dump() (eval_framework.tasks.eval_config.EvalConfig method) module eval_framework eval_framework.base_config eval_framework.context eval_framework.context.determined eval_framework.context.eval eval_framework.context.local eval_framework.evaluation_generator eval_framework.exceptions eval_framework.llm eval_framework.llm.aleph_alpha eval_framework.llm.base eval_framework.llm.huggingface eval_framework.llm.mistral eval_framework.llm.models eval_framework.llm.openai eval_framework.llm.vllm eval_framework.logger eval_framework.main eval_framework.metrics eval_framework.metrics.base eval_framework.metrics.completion eval_framework.metrics.completion.accuracy_completion eval_framework.metrics.completion.aidanbench eval_framework.metrics.completion.bleu eval_framework.metrics.completion.chrf eval_framework.metrics.completion.code_assertion eval_framework.metrics.completion.code_execution_pass_at_one eval_framework.metrics.completion.comet eval_framework.metrics.completion.concordance_index eval_framework.metrics.completion.csv_format eval_framework.metrics.completion.cwe_accuracy eval_framework.metrics.completion.exponential_similarity eval_framework.metrics.completion.f1 eval_framework.metrics.completion.format_checker eval_framework.metrics.completion.grid_difference eval_framework.metrics.completion.ifeval eval_framework.metrics.completion.json_format eval_framework.metrics.completion.language_checker eval_framework.metrics.completion.length_control eval_framework.metrics.completion.math_reasoning_completion eval_framework.metrics.completion.niah_accuracy eval_framework.metrics.completion.placeholder_checker eval_framework.metrics.completion.repetition eval_framework.metrics.completion.rouge_1 eval_framework.metrics.completion.rouge_2 eval_framework.metrics.completion.rouge_geometric_mean eval_framework.metrics.completion.rouge_l eval_framework.metrics.completion.struct_eval_metrics eval_framework.metrics.completion.ter eval_framework.metrics.completion.text_counter eval_framework.metrics.efficiency eval_framework.metrics.efficiency.bytes_per_sequence_position eval_framework.metrics.llm eval_framework.metrics.llm.base eval_framework.metrics.llm.llm_judge_chatbot_style eval_framework.metrics.llm.llm_judge_coherence eval_framework.metrics.llm.llm_judge_completion_accuracy eval_framework.metrics.llm.llm_judge_conciseness eval_framework.metrics.llm.llm_judge_contains_names eval_framework.metrics.llm.llm_judge_format_correctness eval_framework.metrics.llm.llm_judge_instruction eval_framework.metrics.llm.llm_judge_mtbench_pair eval_framework.metrics.llm.llm_judge_mtbench_single eval_framework.metrics.llm.llm_judge_refusal eval_framework.metrics.llm.llm_judge_sql eval_framework.metrics.llm.llm_judge_world_knowledge eval_framework.metrics.llm.utils eval_framework.metrics.loglikelihood eval_framework.metrics.loglikelihood.accuracy_loglikelihood eval_framework.metrics.loglikelihood.base eval_framework.metrics.loglikelihood.confidence_weighted_accuracy eval_framework.metrics.loglikelihood.dcs eval_framework.metrics.loglikelihood.probability_mass eval_framework.metrics.loglikelihood.ternary eval_framework.response_generator eval_framework.result_processors eval_framework.result_processors.base eval_framework.result_processors.hf_uploader eval_framework.result_processors.result_processor eval_framework.result_processors.wandb_uploader eval_framework.run eval_framework.run_direct eval_framework.tasks eval_framework.tasks.base eval_framework.tasks.benchmarks eval_framework.tasks.benchmarks.aidanbench eval_framework.tasks.benchmarks.arc eval_framework.tasks.benchmarks.arc_de eval_framework.tasks.benchmarks.arc_fi eval_framework.tasks.benchmarks.belebele eval_framework.tasks.benchmarks.bigcodebench eval_framework.tasks.benchmarks.casehold eval_framework.tasks.benchmarks.chembench eval_framework.tasks.benchmarks.copa eval_framework.tasks.benchmarks.duc eval_framework.tasks.benchmarks.flores200 eval_framework.tasks.benchmarks.flores_plus eval_framework.tasks.benchmarks.gpqa eval_framework.tasks.benchmarks.gsm8k eval_framework.tasks.benchmarks.hellaswag eval_framework.tasks.benchmarks.hellaswag_de eval_framework.tasks.benchmarks.humaneval eval_framework.tasks.benchmarks.ifeval eval_framework.tasks.benchmarks.include eval_framework.tasks.benchmarks.infinitebench eval_framework.tasks.benchmarks.math_reasoning eval_framework.tasks.benchmarks.mbpp eval_framework.tasks.benchmarks.mmlu eval_framework.tasks.benchmarks.mmlu_de eval_framework.tasks.benchmarks.mmlu_pro eval_framework.tasks.benchmarks.mmmlu eval_framework.tasks.benchmarks.openbookqa eval_framework.tasks.benchmarks.opengptx_eu20 eval_framework.tasks.benchmarks.pawsx eval_framework.tasks.benchmarks.piqa eval_framework.tasks.benchmarks.quality eval_framework.tasks.benchmarks.sciq eval_framework.tasks.benchmarks.sphyr eval_framework.tasks.benchmarks.squad eval_framework.tasks.benchmarks.struct_eval eval_framework.tasks.benchmarks.tablebench eval_framework.tasks.benchmarks.triviaqa eval_framework.tasks.benchmarks.truthfulqa eval_framework.tasks.benchmarks.winogender eval_framework.tasks.benchmarks.winogrande eval_framework.tasks.benchmarks.winox eval_framework.tasks.benchmarks.wmt eval_framework.tasks.benchmarks.zero_scrolls eval_framework.tasks.eval_config eval_framework.tasks.perturbation eval_framework.tasks.registry eval_framework.tasks.task_loader eval_framework.tasks.task_names eval_framework.tasks.utils MOE (eval_framework.tasks.base.Language attribute) MOF (eval_framework.tasks.base.Language attribute) MOG (eval_framework.tasks.base.Language attribute) MOH (eval_framework.tasks.base.Language attribute) MOI (eval_framework.tasks.base.Language attribute) MOJ (eval_framework.tasks.base.Language attribute) MOK (eval_framework.tasks.base.Language attribute) MOL (eval_framework.tasks.base.Language attribute) MOM (eval_framework.tasks.base.Language attribute) MON (eval_framework.tasks.base.Language attribute) MOO (eval_framework.tasks.base.Language attribute) MOP (eval_framework.tasks.base.Language attribute) MOQ (eval_framework.tasks.base.Language attribute) MOR (eval_framework.tasks.base.Language attribute) MOS (eval_framework.tasks.base.Language attribute) MOT (eval_framework.tasks.base.Language attribute) MOU (eval_framework.tasks.base.Language attribute) MOV (eval_framework.tasks.base.Language attribute) MOW (eval_framework.tasks.base.Language attribute) MOX (eval_framework.tasks.base.Language attribute) MOY (eval_framework.tasks.base.Language attribute) MOZ (eval_framework.tasks.base.Language attribute) MPA (eval_framework.tasks.base.Language attribute) MPB (eval_framework.tasks.base.Language attribute) MPC (eval_framework.tasks.base.Language attribute) MPD (eval_framework.tasks.base.Language attribute) MPE (eval_framework.tasks.base.Language attribute) MPF (eval_framework.tasks.base.Language attribute) MPG (eval_framework.tasks.base.Language attribute) MPH (eval_framework.tasks.base.Language attribute) MPI (eval_framework.tasks.base.Language attribute) MPJ (eval_framework.tasks.base.Language attribute) MPK (eval_framework.tasks.base.Language attribute) MPL (eval_framework.tasks.base.Language attribute) MPM (eval_framework.tasks.base.Language attribute) MPN (eval_framework.tasks.base.Language attribute) MPO (eval_framework.tasks.base.Language attribute) MPP (eval_framework.tasks.base.Language attribute) MPQ (eval_framework.tasks.base.Language attribute) MPR (eval_framework.tasks.base.Language attribute) MPS (eval_framework.tasks.base.Language attribute) MPT (eval_framework.tasks.base.Language attribute) MPU (eval_framework.tasks.base.Language attribute) MPV (eval_framework.tasks.base.Language attribute) MPW (eval_framework.tasks.base.Language attribute) MPX (eval_framework.tasks.base.Language attribute) MPY (eval_framework.tasks.base.Language attribute) MPZ (eval_framework.tasks.base.Language attribute) MQA (eval_framework.tasks.base.Language attribute) MQB (eval_framework.tasks.base.Language attribute) MQC (eval_framework.tasks.base.Language attribute) MQD (eval_framework.tasks.base.Language attribute) MQE (eval_framework.tasks.base.Language attribute) MQF (eval_framework.tasks.base.Language attribute) MQG (eval_framework.tasks.base.Language attribute) MQH (eval_framework.tasks.base.Language attribute) MQI (eval_framework.tasks.base.Language attribute) MQJ (eval_framework.tasks.base.Language attribute) MQK (eval_framework.tasks.base.Language attribute) MQL (eval_framework.tasks.base.Language attribute) MQM (eval_framework.tasks.base.Language attribute) MQN (eval_framework.tasks.base.Language attribute) MQO (eval_framework.tasks.base.Language attribute) MQP (eval_framework.tasks.base.Language attribute) MQQ (eval_framework.tasks.base.Language attribute) MQR (eval_framework.tasks.base.Language attribute) MQS (eval_framework.tasks.base.Language attribute) MQT (eval_framework.tasks.base.Language attribute) MQU (eval_framework.tasks.base.Language attribute) MQV (eval_framework.tasks.base.Language attribute) MQW (eval_framework.tasks.base.Language attribute) MQX (eval_framework.tasks.base.Language attribute) MQY (eval_framework.tasks.base.Language attribute) MQZ (eval_framework.tasks.base.Language attribute) MRA (eval_framework.tasks.base.Language attribute) MRB (eval_framework.tasks.base.Language attribute) MRC (eval_framework.tasks.base.Language attribute) MRD (eval_framework.tasks.base.Language attribute) MRE (eval_framework.tasks.base.Language attribute) MRF (eval_framework.tasks.base.Language attribute) MRG (eval_framework.tasks.base.Language attribute) MRH (eval_framework.tasks.base.Language attribute) MRI (eval_framework.tasks.base.Language attribute) MRJ (eval_framework.tasks.base.Language attribute) MRK (eval_framework.tasks.base.Language attribute) MRL (eval_framework.tasks.base.Language attribute) MRM (eval_framework.tasks.base.Language attribute) MRN (eval_framework.tasks.base.Language attribute) MRO (eval_framework.tasks.base.Language attribute) MRP (eval_framework.tasks.base.Language attribute) MRQ (eval_framework.tasks.base.Language attribute) MRR (eval_framework.tasks.base.Language attribute) MRS (eval_framework.tasks.base.Language attribute) MRT (eval_framework.tasks.base.Language attribute) MRU (eval_framework.tasks.base.Language attribute) MRV (eval_framework.tasks.base.Language attribute) MRW (eval_framework.tasks.base.Language attribute) MRX (eval_framework.tasks.base.Language attribute) MRY (eval_framework.tasks.base.Language attribute) MRZ (eval_framework.tasks.base.Language attribute) MSA (eval_framework.tasks.base.Language attribute) MSB (eval_framework.tasks.base.Language attribute) MSC (eval_framework.tasks.base.Language attribute) MSD (eval_framework.tasks.base.Language attribute) MSE (eval_framework.tasks.base.Language attribute) MSF (eval_framework.tasks.base.Language attribute) MSG (eval_framework.tasks.base.Language attribute) MSH (eval_framework.tasks.base.Language attribute) MSI (eval_framework.tasks.base.Language attribute) MSJ (eval_framework.tasks.base.Language attribute) MSK (eval_framework.tasks.base.Language attribute) MSL (eval_framework.tasks.base.Language attribute) MSM (eval_framework.tasks.base.Language attribute) MSN (eval_framework.tasks.base.Language attribute) MSO (eval_framework.tasks.base.Language attribute) MSP (eval_framework.tasks.base.Language attribute) MSQ (eval_framework.tasks.base.Language attribute) MSR (eval_framework.tasks.base.Language attribute) MSS (eval_framework.tasks.base.Language attribute) MST (eval_framework.tasks.base.Language attribute) MSU (eval_framework.tasks.base.Language attribute) MSV (eval_framework.tasks.base.Language attribute) MSW (eval_framework.tasks.base.Language attribute) MSX (eval_framework.tasks.base.Language attribute) MSY (eval_framework.tasks.base.Language attribute) MSZ (eval_framework.tasks.base.Language attribute) MTA (eval_framework.tasks.base.Language attribute) MTB (eval_framework.tasks.base.Language attribute) MTBenchJudgePair (class in eval_framework.metrics.llm.llm_judge_mtbench_pair) MTBenchJudgePairMetricContext (class in eval_framework.metrics.llm.llm_judge_mtbench_pair) MTBenchJudgeSingle (class in eval_framework.metrics.llm.llm_judge_mtbench_single) MTBenchJudgeSingleMetricContext (class in eval_framework.metrics.llm.llm_judge_mtbench_single) MTC (eval_framework.tasks.base.Language attribute) MTD (eval_framework.tasks.base.Language attribute) MTE (eval_framework.tasks.base.Language attribute) MTF (eval_framework.tasks.base.Language attribute) MTG (eval_framework.tasks.base.Language attribute) MTH (eval_framework.tasks.base.Language attribute) MTI (eval_framework.tasks.base.Language attribute) MTJ (eval_framework.tasks.base.Language attribute) MTK (eval_framework.tasks.base.Language attribute) MTL (eval_framework.tasks.base.Language attribute) MTM (eval_framework.tasks.base.Language attribute) MTN (eval_framework.tasks.base.Language attribute) MTO (eval_framework.tasks.base.Language attribute) MTP (eval_framework.tasks.base.Language attribute) MTQ (eval_framework.tasks.base.Language attribute) MTR (eval_framework.tasks.base.Language attribute) MTS (eval_framework.tasks.base.Language attribute) MTT (eval_framework.tasks.base.Language attribute) MTU (eval_framework.tasks.base.Language attribute) MTV (eval_framework.tasks.base.Language attribute) MTW (eval_framework.tasks.base.Language attribute) MTX (eval_framework.tasks.base.Language attribute) MTY (eval_framework.tasks.base.Language attribute) MTZ (eval_framework.tasks.base.Language attribute) MUA (eval_framework.tasks.base.Language attribute) MUB (eval_framework.tasks.base.Language attribute) MUC (eval_framework.tasks.base.Language attribute) MUD (eval_framework.tasks.base.Language attribute) MUE (eval_framework.tasks.base.Language attribute) MUG (eval_framework.tasks.base.Language attribute) MUH (eval_framework.tasks.base.Language attribute) MUI (eval_framework.tasks.base.Language attribute) MUJ (eval_framework.tasks.base.Language attribute) MUK (eval_framework.tasks.base.Language attribute) MUL (eval_framework.tasks.base.Language attribute) MUM (eval_framework.tasks.base.Language attribute) MUO (eval_framework.tasks.base.Language attribute) MUP (eval_framework.tasks.base.Language attribute) MUQ (eval_framework.tasks.base.Language attribute) MUR (eval_framework.tasks.base.Language attribute) MUS (eval_framework.tasks.base.Language attribute) MUT (eval_framework.tasks.base.Language attribute) MUU (eval_framework.tasks.base.Language attribute) MUV (eval_framework.tasks.base.Language attribute) MUW (eval_framework.tasks.base.Language attribute) MUX (eval_framework.tasks.base.Language attribute) MUY (eval_framework.tasks.base.Language attribute) MUZ (eval_framework.tasks.base.Language attribute) MVA (eval_framework.tasks.base.Language attribute) MVB (eval_framework.tasks.base.Language attribute) MVC (eval_framework.tasks.base.Language attribute) MVD (eval_framework.tasks.base.Language attribute) MVE (eval_framework.tasks.base.Language attribute) MVF (eval_framework.tasks.base.Language attribute) MVG (eval_framework.tasks.base.Language attribute) MVH (eval_framework.tasks.base.Language attribute) MVI (eval_framework.tasks.base.Language attribute) MVJ (eval_framework.tasks.base.Language attribute) MVK (eval_framework.tasks.base.Language attribute) MVL (eval_framework.tasks.base.Language attribute) MVM (eval_framework.tasks.base.Language attribute) MVN (eval_framework.tasks.base.Language attribute) MVO (eval_framework.tasks.base.Language attribute) MVP (eval_framework.tasks.base.Language attribute) MVQ (eval_framework.tasks.base.Language attribute) MVR (eval_framework.tasks.base.Language attribute) MVS (eval_framework.tasks.base.Language attribute) MVT (eval_framework.tasks.base.Language attribute) MVU (eval_framework.tasks.base.Language attribute) MVV (eval_framework.tasks.base.Language attribute) MVW (eval_framework.tasks.base.Language attribute) MVX (eval_framework.tasks.base.Language attribute) MVY (eval_framework.tasks.base.Language attribute) MVZ (eval_framework.tasks.base.Language attribute) MWA (eval_framework.tasks.base.Language attribute) MWB (eval_framework.tasks.base.Language attribute) MWC (eval_framework.tasks.base.Language attribute) MWD (eval_framework.tasks.base.Language attribute) MWE (eval_framework.tasks.base.Language attribute) MWF (eval_framework.tasks.base.Language attribute) MWG (eval_framework.tasks.base.Language attribute) MWH (eval_framework.tasks.base.Language attribute) MWI (eval_framework.tasks.base.Language attribute) MWJ (eval_framework.tasks.base.Language attribute) MWK (eval_framework.tasks.base.Language attribute) MWL (eval_framework.tasks.base.Language attribute) MWM (eval_framework.tasks.base.Language attribute) MWN (eval_framework.tasks.base.Language attribute) MWO (eval_framework.tasks.base.Language attribute) MWP (eval_framework.tasks.base.Language attribute) MWQ (eval_framework.tasks.base.Language attribute) MWR (eval_framework.tasks.base.Language attribute) MWS (eval_framework.tasks.base.Language attribute) MWT (eval_framework.tasks.base.Language attribute) MWU (eval_framework.tasks.base.Language attribute) MWV (eval_framework.tasks.base.Language attribute) MWW (eval_framework.tasks.base.Language attribute) MWX (eval_framework.tasks.base.Language attribute) MWY (eval_framework.tasks.base.Language attribute) MWZ (eval_framework.tasks.base.Language attribute) MXA (eval_framework.tasks.base.Language attribute) MXB (eval_framework.tasks.base.Language attribute) MXC (eval_framework.tasks.base.Language attribute) MXD (eval_framework.tasks.base.Language attribute) MXE (eval_framework.tasks.base.Language attribute) MXF (eval_framework.tasks.base.Language attribute) MXG (eval_framework.tasks.base.Language attribute) MXH (eval_framework.tasks.base.Language attribute) MXI (eval_framework.tasks.base.Language attribute) MXJ (eval_framework.tasks.base.Language attribute) MXK (eval_framework.tasks.base.Language attribute) MXL (eval_framework.tasks.base.Language attribute) MXM (eval_framework.tasks.base.Language attribute) MXN (eval_framework.tasks.base.Language attribute) MXO (eval_framework.tasks.base.Language attribute) MXP (eval_framework.tasks.base.Language attribute) MXQ (eval_framework.tasks.base.Language attribute) MXR (eval_framework.tasks.base.Language attribute) MXS (eval_framework.tasks.base.Language attribute) MXT (eval_framework.tasks.base.Language attribute) MXU (eval_framework.tasks.base.Language attribute) MXV (eval_framework.tasks.base.Language attribute) MXW (eval_framework.tasks.base.Language attribute) MXX (eval_framework.tasks.base.Language attribute) MXY (eval_framework.tasks.base.Language attribute) MXZ (eval_framework.tasks.base.Language attribute) MYA (eval_framework.tasks.base.Language attribute) MYB (eval_framework.tasks.base.Language attribute) MYC (eval_framework.tasks.base.Language attribute) MYD (eval_framework.tasks.base.Language attribute) MYE (eval_framework.tasks.base.Language attribute) MYF (eval_framework.tasks.base.Language attribute) MYG (eval_framework.tasks.base.Language attribute) MYH (eval_framework.tasks.base.Language attribute) MYI (eval_framework.tasks.base.Language attribute) MYJ (eval_framework.tasks.base.Language attribute) MYK (eval_framework.tasks.base.Language attribute) MYL (eval_framework.tasks.base.Language attribute) MYM (eval_framework.tasks.base.Language attribute) MYO (eval_framework.tasks.base.Language attribute) MYP (eval_framework.tasks.base.Language attribute) MYQ (eval_framework.tasks.base.Language attribute) MYR (eval_framework.tasks.base.Language attribute) MYS (eval_framework.tasks.base.Language attribute) mysql (eval_framework.metrics.llm.llm_judge_sql.SqlDialects attribute) MYT (eval_framework.tasks.base.Language attribute) MYU (eval_framework.tasks.base.Language attribute) MYV (eval_framework.tasks.base.Language attribute) MYW (eval_framework.tasks.base.Language attribute) MYX (eval_framework.tasks.base.Language attribute) MYY (eval_framework.tasks.base.Language attribute) MYZ (eval_framework.tasks.base.Language attribute) MZA (eval_framework.tasks.base.Language attribute) MZB (eval_framework.tasks.base.Language attribute) MZC (eval_framework.tasks.base.Language attribute) MZD (eval_framework.tasks.base.Language attribute) MZE (eval_framework.tasks.base.Language attribute) MZF (eval_framework.tasks.base.Language attribute) MZG (eval_framework.tasks.base.Language attribute) MZH (eval_framework.tasks.base.Language attribute) MZI (eval_framework.tasks.base.Language attribute) MZJ (eval_framework.tasks.base.Language attribute) MZK (eval_framework.tasks.base.Language attribute) MZL (eval_framework.tasks.base.Language attribute) MZM (eval_framework.tasks.base.Language attribute) MZN (eval_framework.tasks.base.Language attribute) MZO (eval_framework.tasks.base.Language attribute) MZP (eval_framework.tasks.base.Language attribute) MZQ (eval_framework.tasks.base.Language attribute) MZR (eval_framework.tasks.base.Language attribute) MZS (eval_framework.tasks.base.Language attribute) MZT (eval_framework.tasks.base.Language attribute) MZU (eval_framework.tasks.base.Language attribute) MZV (eval_framework.tasks.base.Language attribute) MZW (eval_framework.tasks.base.Language attribute) MZX (eval_framework.tasks.base.Language attribute) MZY (eval_framework.tasks.base.Language attribute) MZZ (eval_framework.tasks.base.Language attribute) N NAA (eval_framework.tasks.base.Language attribute) NAB (eval_framework.tasks.base.Language attribute) NAC (eval_framework.tasks.base.Language attribute) NAD (eval_framework.tasks.base.Language attribute) NAE (eval_framework.tasks.base.Language attribute) NAF (eval_framework.tasks.base.Language attribute) NAG (eval_framework.tasks.base.Language attribute) NAJ (eval_framework.tasks.base.Language attribute) NAK (eval_framework.tasks.base.Language attribute) NAL (eval_framework.tasks.base.Language attribute) NAM (eval_framework.tasks.base.Language attribute) name (eval_framework.llm.base.BaseLLM property) (eval_framework.llm.huggingface.HFLLM property) (eval_framework.llm.vllm.BaseVLLMModel property) NAME (eval_framework.metrics.base.BaseMetric attribute) (eval_framework.metrics.completion.accuracy_completion.AccuracyCompletion attribute) (eval_framework.metrics.completion.aidanbench.AidanBenchMetric attribute) (eval_framework.metrics.completion.bleu.BLEU attribute) (eval_framework.metrics.completion.bleu.LINEWISE_BLEU attribute) (eval_framework.metrics.completion.bleu.ResponseToOriginalBLEU attribute) (eval_framework.metrics.completion.chrf.CHRF attribute) (eval_framework.metrics.completion.chrf.LINEWISE_CHRF attribute) (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOne attribute) (eval_framework.metrics.completion.comet.COMET attribute) (eval_framework.metrics.completion.concordance_index.ConcordanceIndex attribute) (eval_framework.metrics.completion.csv_format.CSVFormat attribute) (eval_framework.metrics.completion.cwe_accuracy.CWEAccuracy attribute) (eval_framework.metrics.completion.exponential_similarity.ExponentialSimilarity attribute) (eval_framework.metrics.completion.f1.F1 attribute) (eval_framework.metrics.completion.format_checker.CheckJsonFormat attribute) (eval_framework.metrics.completion.format_checker.CheckPostScriptFormat attribute) (eval_framework.metrics.completion.grid_difference.GridDifference attribute) (eval_framework.metrics.completion.ifeval.IFEvalMetric attribute) (eval_framework.metrics.completion.json_format.JsonFormat attribute) (eval_framework.metrics.completion.language_checker.GermanCompletionChecker attribute) (eval_framework.metrics.completion.language_checker.LanguageChecker attribute) (eval_framework.metrics.completion.language_checker.LanguageConsistencyChecker attribute) (eval_framework.metrics.completion.language_checker.LanguageRawConsistencyChecker attribute) (eval_framework.metrics.completion.length_control.LengthControl attribute) (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) (eval_framework.metrics.completion.niah_accuracy.NIAHAccuracy attribute) (eval_framework.metrics.completion.placeholder_checker.PlaceholderChecker attribute) (eval_framework.metrics.completion.repetition.WordRepetition attribute) (eval_framework.metrics.completion.rouge_1.ROUGE_1 attribute) (eval_framework.metrics.completion.rouge_2.ROUGE_2 attribute) (eval_framework.metrics.completion.rouge_geometric_mean.ROUGE_GEOMETRIC_MEAN attribute) (eval_framework.metrics.completion.rouge_l.ROUGE_L attribute) (eval_framework.metrics.completion.struct_eval_metrics.RenderableStructMetric attribute) (eval_framework.metrics.completion.struct_eval_metrics.StructMetric attribute) (eval_framework.metrics.completion.ter.LINEWISE_TER attribute) (eval_framework.metrics.completion.ter.TER attribute) (eval_framework.metrics.completion.text_counter.ParagraphCounter attribute) (eval_framework.metrics.completion.text_counter.ResponseToOriginalLengthRatio attribute) (eval_framework.metrics.completion.text_counter.SentenceCounter attribute) (eval_framework.metrics.completion.text_counter.WordCounter attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesCompletion attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesLoglikelihood attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsCompletion attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsLoglikelihood attribute) (eval_framework.metrics.llm.llm_judge_chatbot_style.LLMJudgeChatbotStyle attribute) (eval_framework.metrics.llm.llm_judge_coherence.LLMJudgeCoherence attribute) (eval_framework.metrics.llm.llm_judge_completion_accuracy.LLMJudgeCompletionAccuracy attribute) (eval_framework.metrics.llm.llm_judge_conciseness.LLMJudgeConciseness attribute) (eval_framework.metrics.llm.llm_judge_contains_names.LLMJudgeAvoidsNames attribute) (eval_framework.metrics.llm.llm_judge_format_correctness.LLMJudgeFormatCorrectness attribute) (eval_framework.metrics.llm.llm_judge_instruction.LLMJudgeInstruction attribute) (eval_framework.metrics.llm.llm_judge_mtbench_pair.MTBenchJudgePair attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.MTBenchJudgeSingle attribute) (eval_framework.metrics.llm.llm_judge_refusal.LLMJudgeRefusal attribute) (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql attribute) (eval_framework.metrics.llm.llm_judge_world_knowledge.LLMJudgeWorldKnowledge attribute) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyLoglikelihood attribute) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyNormLoglikelihood attribute) (eval_framework.metrics.loglikelihood.confidence_weighted_accuracy.ConfidenceWeightedAccuracy attribute) (eval_framework.metrics.loglikelihood.dcs.DistributionalCorrectnessScore attribute) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMass attribute) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMassNorm attribute) (eval_framework.metrics.loglikelihood.ternary.TernaryScore attribute) (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc.ARC_IDK attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBenchHard attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBenchHardInstruct attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBenchInstruct attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.copa.COPA_IDK attribute) (eval_framework.tasks.benchmarks.duc.DUC_ABSTRACTIVE attribute) (eval_framework.tasks.benchmarks.duc.DUC_EXTRACTIVE attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA_COT attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA_IDK attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8K attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG_IDK attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEvalInstruct attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalDe attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalFiSv attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeDebug attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeRun attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnDia attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnMC attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnQA attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_MathFind attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveKV2 attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveNumber attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrievePassKey1 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2025 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATHLvl5 attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP_PROMPT_WITHOUT_TESTS attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP_PROMPT_WITHOUT_TESTS_SANITIZED attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP_SANITIZED attribute) (eval_framework.tasks.benchmarks.mmlu.FullTextMMLU attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_IDK attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA_EVAL_HARNESS attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA_IDK attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.TRUTHFULQA_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.TRUTHFULQA_EU20_FR attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.piqa.PIQA_IDK attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ_IDK attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness_IDK attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.RenderableStructEval attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA_IDK attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER_IDK attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE_IDK attribute) (eval_framework.tasks.benchmarks.winox.WINOX_DE attribute) (eval_framework.tasks.benchmarks.winox.WINOX_FR attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.wmt.WMT14 attribute) (eval_framework.tasks.benchmarks.wmt.WMT14_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT16 attribute) (eval_framework.tasks.benchmarks.wmt.WMT16_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT20 attribute) (eval_framework.tasks.benchmarks.wmt.WMT20_INSTRUCT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_GOV_REPORT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_MUSIQUE attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_NARRATIVEQA attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QASPER attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QMSUM attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SPACE_DIGEST attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SQUALITY attribute) NAMES (eval_framework.metrics.base.BaseMetric attribute) NAN (eval_framework.tasks.base.Language attribute) NAO (eval_framework.tasks.base.Language attribute) NAP (eval_framework.tasks.base.Language attribute) NAQ (eval_framework.tasks.base.Language attribute) NAR (eval_framework.tasks.base.Language attribute) NAS (eval_framework.tasks.base.Language attribute) NAT (eval_framework.tasks.base.Language attribute) NAU (eval_framework.tasks.base.Language attribute) NAV (eval_framework.tasks.base.Language attribute) NAW (eval_framework.tasks.base.Language attribute) NAX (eval_framework.tasks.base.Language attribute) NAY (eval_framework.tasks.base.Language attribute) NAZ (eval_framework.tasks.base.Language attribute) NBA (eval_framework.tasks.base.Language attribute) NBB (eval_framework.tasks.base.Language attribute) NBC (eval_framework.tasks.base.Language attribute) NBD (eval_framework.tasks.base.Language attribute) NBE (eval_framework.tasks.base.Language attribute) NBF (eval_framework.tasks.base.Language attribute) NBG (eval_framework.tasks.base.Language attribute) NBH (eval_framework.tasks.base.Language attribute) NBI (eval_framework.tasks.base.Language attribute) NBJ (eval_framework.tasks.base.Language attribute) NBK (eval_framework.tasks.base.Language attribute) NBL (eval_framework.tasks.base.Language attribute) NBM (eval_framework.tasks.base.Language attribute) NBN (eval_framework.tasks.base.Language attribute) NBO (eval_framework.tasks.base.Language attribute) NBP (eval_framework.tasks.base.Language attribute) NBQ (eval_framework.tasks.base.Language attribute) NBR (eval_framework.tasks.base.Language attribute) NBS (eval_framework.tasks.base.Language attribute) NBT (eval_framework.tasks.base.Language attribute) NBU (eval_framework.tasks.base.Language attribute) NBV (eval_framework.tasks.base.Language attribute) NBW (eval_framework.tasks.base.Language attribute) NBX (eval_framework.tasks.base.Language attribute) NBY (eval_framework.tasks.base.Language attribute) NCA (eval_framework.tasks.base.Language attribute) NCB (eval_framework.tasks.base.Language attribute) NCC (eval_framework.tasks.base.Language attribute) NCD (eval_framework.tasks.base.Language attribute) NCE (eval_framework.tasks.base.Language attribute) NCF (eval_framework.tasks.base.Language attribute) NCG (eval_framework.tasks.base.Language attribute) NCH (eval_framework.tasks.base.Language attribute) NCI (eval_framework.tasks.base.Language attribute) NCJ (eval_framework.tasks.base.Language attribute) NCK (eval_framework.tasks.base.Language attribute) NCL (eval_framework.tasks.base.Language attribute) NCM (eval_framework.tasks.base.Language attribute) NCN (eval_framework.tasks.base.Language attribute) NCO (eval_framework.tasks.base.Language attribute) NCP (eval_framework.tasks.base.Language attribute) NCQ (eval_framework.tasks.base.Language attribute) NCR (eval_framework.tasks.base.Language attribute) NCS (eval_framework.tasks.base.Language attribute) NCT (eval_framework.tasks.base.Language attribute) NCU (eval_framework.tasks.base.Language attribute) NCX (eval_framework.tasks.base.Language attribute) NCZ (eval_framework.tasks.base.Language attribute) NDA (eval_framework.tasks.base.Language attribute) NDB (eval_framework.tasks.base.Language attribute) NDC (eval_framework.tasks.base.Language attribute) NDD (eval_framework.tasks.base.Language attribute) NDE (eval_framework.tasks.base.Language attribute) NDF (eval_framework.tasks.base.Language attribute) NDG (eval_framework.tasks.base.Language attribute) NDH (eval_framework.tasks.base.Language attribute) NDI (eval_framework.tasks.base.Language attribute) NDJ (eval_framework.tasks.base.Language attribute) NDK (eval_framework.tasks.base.Language attribute) NDL (eval_framework.tasks.base.Language attribute) NDM (eval_framework.tasks.base.Language attribute) NDN (eval_framework.tasks.base.Language attribute) NDO (eval_framework.tasks.base.Language attribute) NDP (eval_framework.tasks.base.Language attribute) NDQ (eval_framework.tasks.base.Language attribute) NDR (eval_framework.tasks.base.Language attribute) NDS (eval_framework.tasks.base.Language attribute) NDT (eval_framework.tasks.base.Language attribute) NDU (eval_framework.tasks.base.Language attribute) NDV (eval_framework.tasks.base.Language attribute) NDW (eval_framework.tasks.base.Language attribute) NDX (eval_framework.tasks.base.Language attribute) NDY (eval_framework.tasks.base.Language attribute) NDZ (eval_framework.tasks.base.Language attribute) NEA (eval_framework.tasks.base.Language attribute) NEB (eval_framework.tasks.base.Language attribute) NEC (eval_framework.tasks.base.Language attribute) NED (eval_framework.tasks.base.Language attribute) NEE (eval_framework.tasks.base.Language attribute) NEF (eval_framework.tasks.base.Language attribute) NEG (eval_framework.tasks.base.Language attribute) NEH (eval_framework.tasks.base.Language attribute) NEI (eval_framework.tasks.base.Language attribute) NEJ (eval_framework.tasks.base.Language attribute) NEK (eval_framework.tasks.base.Language attribute) NEM (eval_framework.tasks.base.Language attribute) NEN (eval_framework.tasks.base.Language attribute) NEO (eval_framework.tasks.base.Language attribute) NEP (eval_framework.tasks.base.Language attribute) NEQ (eval_framework.tasks.base.Language attribute) NER (eval_framework.tasks.base.Language attribute) NES (eval_framework.tasks.base.Language attribute) NET (eval_framework.tasks.base.Language attribute) NEU (eval_framework.tasks.base.Language attribute) NEV (eval_framework.tasks.base.Language attribute) NEW (eval_framework.tasks.base.Language attribute) NEX (eval_framework.tasks.base.Language attribute) NEY (eval_framework.tasks.base.Language attribute) NEZ (eval_framework.tasks.base.Language attribute) NFA (eval_framework.tasks.base.Language attribute) NFD (eval_framework.tasks.base.Language attribute) NFG (eval_framework.tasks.base.Language attribute) NFK (eval_framework.tasks.base.Language attribute) NFL (eval_framework.tasks.base.Language attribute) NFR (eval_framework.tasks.base.Language attribute) NFU (eval_framework.tasks.base.Language attribute) NGA (eval_framework.tasks.base.Language attribute) NGB (eval_framework.tasks.base.Language attribute) NGC (eval_framework.tasks.base.Language attribute) NGD (eval_framework.tasks.base.Language attribute) NGE (eval_framework.tasks.base.Language attribute) NGG (eval_framework.tasks.base.Language attribute) NGH (eval_framework.tasks.base.Language attribute) NGI (eval_framework.tasks.base.Language attribute) NGJ (eval_framework.tasks.base.Language attribute) NGK (eval_framework.tasks.base.Language attribute) NGL (eval_framework.tasks.base.Language attribute) NGM (eval_framework.tasks.base.Language attribute) NGN (eval_framework.tasks.base.Language attribute) NGO (eval_framework.tasks.base.Language attribute) NGP (eval_framework.tasks.base.Language attribute) NGQ (eval_framework.tasks.base.Language attribute) NGR (eval_framework.tasks.base.Language attribute) NGS (eval_framework.tasks.base.Language attribute) NGT (eval_framework.tasks.base.Language attribute) NGU (eval_framework.tasks.base.Language attribute) NGV (eval_framework.tasks.base.Language attribute) NGW (eval_framework.tasks.base.Language attribute) NGX (eval_framework.tasks.base.Language attribute) NGY (eval_framework.tasks.base.Language attribute) NGZ (eval_framework.tasks.base.Language attribute) NHA (eval_framework.tasks.base.Language attribute) NHB (eval_framework.tasks.base.Language attribute) NHC (eval_framework.tasks.base.Language attribute) NHD (eval_framework.tasks.base.Language attribute) NHE (eval_framework.tasks.base.Language attribute) NHF (eval_framework.tasks.base.Language attribute) NHG (eval_framework.tasks.base.Language attribute) NHH (eval_framework.tasks.base.Language attribute) NHI (eval_framework.tasks.base.Language attribute) NHJ (eval_framework.tasks.base.Language attribute) NHK (eval_framework.tasks.base.Language attribute) NHM (eval_framework.tasks.base.Language attribute) NHN (eval_framework.tasks.base.Language attribute) NHO (eval_framework.tasks.base.Language attribute) NHP (eval_framework.tasks.base.Language attribute) NHQ (eval_framework.tasks.base.Language attribute) NHR (eval_framework.tasks.base.Language attribute) NHS (eval_framework.tasks.base.Language attribute) NHT (eval_framework.tasks.base.Language attribute) NHU (eval_framework.tasks.base.Language attribute) NHV (eval_framework.tasks.base.Language attribute) NHW (eval_framework.tasks.base.Language attribute) NHX (eval_framework.tasks.base.Language attribute) NHY (eval_framework.tasks.base.Language attribute) NHZ (eval_framework.tasks.base.Language attribute) NIA (eval_framework.tasks.base.Language attribute) NIAHAccuracy (class in eval_framework.metrics.completion.niah_accuracy) NIB (eval_framework.tasks.base.Language attribute) NID (eval_framework.tasks.base.Language attribute) NIE (eval_framework.tasks.base.Language attribute) NIF (eval_framework.tasks.base.Language attribute) NIG (eval_framework.tasks.base.Language attribute) NIH (eval_framework.tasks.base.Language attribute) NII (eval_framework.tasks.base.Language attribute) NIJ (eval_framework.tasks.base.Language attribute) NIK (eval_framework.tasks.base.Language attribute) NIL (eval_framework.tasks.base.Language attribute) NIM (eval_framework.tasks.base.Language attribute) NIN (eval_framework.tasks.base.Language attribute) NIO (eval_framework.tasks.base.Language attribute) NIQ (eval_framework.tasks.base.Language attribute) NIR (eval_framework.tasks.base.Language attribute) NIS (eval_framework.tasks.base.Language attribute) NIT (eval_framework.tasks.base.Language attribute) NIU (eval_framework.tasks.base.Language attribute) NIV (eval_framework.tasks.base.Language attribute) NIW (eval_framework.tasks.base.Language attribute) NIX (eval_framework.tasks.base.Language attribute) NIY (eval_framework.tasks.base.Language attribute) NIZ (eval_framework.tasks.base.Language attribute) NJA (eval_framework.tasks.base.Language attribute) NJB (eval_framework.tasks.base.Language attribute) NJD (eval_framework.tasks.base.Language attribute) NJH (eval_framework.tasks.base.Language attribute) NJI (eval_framework.tasks.base.Language attribute) NJJ (eval_framework.tasks.base.Language attribute) NJL (eval_framework.tasks.base.Language attribute) NJM (eval_framework.tasks.base.Language attribute) NJN (eval_framework.tasks.base.Language attribute) NJO (eval_framework.tasks.base.Language attribute) NJR (eval_framework.tasks.base.Language attribute) NJS (eval_framework.tasks.base.Language attribute) NJT (eval_framework.tasks.base.Language attribute) NJU (eval_framework.tasks.base.Language attribute) NJX (eval_framework.tasks.base.Language attribute) NJY (eval_framework.tasks.base.Language attribute) NJZ (eval_framework.tasks.base.Language attribute) NKA (eval_framework.tasks.base.Language attribute) NKB (eval_framework.tasks.base.Language attribute) NKC (eval_framework.tasks.base.Language attribute) NKD (eval_framework.tasks.base.Language attribute) NKE (eval_framework.tasks.base.Language attribute) NKF (eval_framework.tasks.base.Language attribute) NKG (eval_framework.tasks.base.Language attribute) NKH (eval_framework.tasks.base.Language attribute) NKI (eval_framework.tasks.base.Language attribute) NKJ (eval_framework.tasks.base.Language attribute) NKK (eval_framework.tasks.base.Language attribute) NKM (eval_framework.tasks.base.Language attribute) NKN (eval_framework.tasks.base.Language attribute) NKO (eval_framework.tasks.base.Language attribute) NKP (eval_framework.tasks.base.Language attribute) NKQ (eval_framework.tasks.base.Language attribute) NKR (eval_framework.tasks.base.Language attribute) NKS (eval_framework.tasks.base.Language attribute) NKT (eval_framework.tasks.base.Language attribute) NKU (eval_framework.tasks.base.Language attribute) NKV (eval_framework.tasks.base.Language attribute) NKW (eval_framework.tasks.base.Language attribute) NKX (eval_framework.tasks.base.Language attribute) NKY (eval_framework.tasks.base.Language attribute) NKZ (eval_framework.tasks.base.Language attribute) NLA (eval_framework.tasks.base.Language attribute) NLC (eval_framework.tasks.base.Language attribute) NLD (eval_framework.tasks.base.Language attribute) NLE (eval_framework.tasks.base.Language attribute) NLG (eval_framework.tasks.base.Language attribute) NLI (eval_framework.tasks.base.Language attribute) NLJ (eval_framework.tasks.base.Language attribute) NLK (eval_framework.tasks.base.Language attribute) NLL (eval_framework.tasks.base.Language attribute) NLM (eval_framework.tasks.base.Language attribute) NLN (eval_framework.tasks.base.Language attribute) NLO (eval_framework.tasks.base.Language attribute) NLQ (eval_framework.tasks.base.Language attribute) NLR (eval_framework.tasks.base.Language attribute) NLU (eval_framework.tasks.base.Language attribute) NLV (eval_framework.tasks.base.Language attribute) NLW (eval_framework.tasks.base.Language attribute) NLX (eval_framework.tasks.base.Language attribute) NLY (eval_framework.tasks.base.Language attribute) NLZ (eval_framework.tasks.base.Language attribute) NMA (eval_framework.tasks.base.Language attribute) NMB (eval_framework.tasks.base.Language attribute) NMC (eval_framework.tasks.base.Language attribute) NMD (eval_framework.tasks.base.Language attribute) NME (eval_framework.tasks.base.Language attribute) NMF (eval_framework.tasks.base.Language attribute) NMG (eval_framework.tasks.base.Language attribute) NMH (eval_framework.tasks.base.Language attribute) NMI (eval_framework.tasks.base.Language attribute) NMJ (eval_framework.tasks.base.Language attribute) NMK (eval_framework.tasks.base.Language attribute) NML (eval_framework.tasks.base.Language attribute) NMM (eval_framework.tasks.base.Language attribute) NMN (eval_framework.tasks.base.Language attribute) NMO (eval_framework.tasks.base.Language attribute) NMP (eval_framework.tasks.base.Language attribute) NMQ (eval_framework.tasks.base.Language attribute) NMR (eval_framework.tasks.base.Language attribute) NMS (eval_framework.tasks.base.Language attribute) NMT (eval_framework.tasks.base.Language attribute) NMU (eval_framework.tasks.base.Language attribute) NMV (eval_framework.tasks.base.Language attribute) NMW (eval_framework.tasks.base.Language attribute) NMX (eval_framework.tasks.base.Language attribute) NMY (eval_framework.tasks.base.Language attribute) NMZ (eval_framework.tasks.base.Language attribute) NNA (eval_framework.tasks.base.Language attribute) NNB (eval_framework.tasks.base.Language attribute) NNC (eval_framework.tasks.base.Language attribute) NND (eval_framework.tasks.base.Language attribute) NNE (eval_framework.tasks.base.Language attribute) NNF (eval_framework.tasks.base.Language attribute) NNG (eval_framework.tasks.base.Language attribute) NNH (eval_framework.tasks.base.Language attribute) NNI (eval_framework.tasks.base.Language attribute) NNJ (eval_framework.tasks.base.Language attribute) NNK (eval_framework.tasks.base.Language attribute) NNL (eval_framework.tasks.base.Language attribute) NNM (eval_framework.tasks.base.Language attribute) NNN (eval_framework.tasks.base.Language attribute) NNO (eval_framework.tasks.base.Language attribute) NNP (eval_framework.tasks.base.Language attribute) NNQ (eval_framework.tasks.base.Language attribute) NNR (eval_framework.tasks.base.Language attribute) NNS (eval_framework.tasks.base.Language attribute) NNT (eval_framework.tasks.base.Language attribute) NNU (eval_framework.tasks.base.Language attribute) NNV (eval_framework.tasks.base.Language attribute) NNW (eval_framework.tasks.base.Language attribute) NNX (eval_framework.tasks.base.Language attribute) NNY (eval_framework.tasks.base.Language attribute) NNZ (eval_framework.tasks.base.Language attribute) NOA (eval_framework.tasks.base.Language attribute) NOB (eval_framework.tasks.base.Language attribute) NOC (eval_framework.tasks.base.Language attribute) NOD (eval_framework.tasks.base.Language attribute) NOE (eval_framework.tasks.base.Language attribute) NOF (eval_framework.tasks.base.Language attribute) NOG (eval_framework.tasks.base.Language attribute) NOH (eval_framework.tasks.base.Language attribute) NOI (eval_framework.tasks.base.Language attribute) NOJ (eval_framework.tasks.base.Language attribute) NOK (eval_framework.tasks.base.Language attribute) NOL (eval_framework.tasks.base.Language attribute) NOM (eval_framework.tasks.base.Language attribute) NON (eval_framework.tasks.base.Language attribute) NOO (eval_framework.tasks.base.Language attribute) NOP (eval_framework.tasks.base.Language attribute) NOQ (eval_framework.tasks.base.Language attribute) NOR (eval_framework.tasks.base.Language attribute) normalize_expression() (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion method) NOS (eval_framework.tasks.base.Language attribute) NOT (eval_framework.tasks.base.Language attribute) NOU (eval_framework.tasks.base.Language attribute) NOV (eval_framework.tasks.base.Language attribute) NOW (eval_framework.tasks.base.Language attribute) NOY (eval_framework.tasks.base.Language attribute) NOZ (eval_framework.tasks.base.Language attribute) NPA (eval_framework.tasks.base.Language attribute) NPB (eval_framework.tasks.base.Language attribute) NPG (eval_framework.tasks.base.Language attribute) NPH (eval_framework.tasks.base.Language attribute) NPI (eval_framework.tasks.base.Language attribute) NPL (eval_framework.tasks.base.Language attribute) NPN (eval_framework.tasks.base.Language attribute) NPO (eval_framework.tasks.base.Language attribute) NPS (eval_framework.tasks.base.Language attribute) NPU (eval_framework.tasks.base.Language attribute) NPX (eval_framework.tasks.base.Language attribute) NPY (eval_framework.tasks.base.Language attribute) NQG (eval_framework.tasks.base.Language attribute) NQK (eval_framework.tasks.base.Language attribute) NQL (eval_framework.tasks.base.Language attribute) NQM (eval_framework.tasks.base.Language attribute) NQN (eval_framework.tasks.base.Language attribute) NQO (eval_framework.tasks.base.Language attribute) NQQ (eval_framework.tasks.base.Language attribute) NQT (eval_framework.tasks.base.Language attribute) NQY (eval_framework.tasks.base.Language attribute) NRA (eval_framework.tasks.base.Language attribute) NRB (eval_framework.tasks.base.Language attribute) NRC (eval_framework.tasks.base.Language attribute) NRE (eval_framework.tasks.base.Language attribute) NRF (eval_framework.tasks.base.Language attribute) NRG (eval_framework.tasks.base.Language attribute) NRI (eval_framework.tasks.base.Language attribute) NRK (eval_framework.tasks.base.Language attribute) NRL (eval_framework.tasks.base.Language attribute) NRM (eval_framework.tasks.base.Language attribute) NRN (eval_framework.tasks.base.Language attribute) NRP (eval_framework.tasks.base.Language attribute) NRR (eval_framework.tasks.base.Language attribute) NRT (eval_framework.tasks.base.Language attribute) NRU (eval_framework.tasks.base.Language attribute) NRX (eval_framework.tasks.base.Language attribute) NRZ (eval_framework.tasks.base.Language attribute) NSA (eval_framework.tasks.base.Language attribute) NSB (eval_framework.tasks.base.Language attribute) NSC (eval_framework.tasks.base.Language attribute) NSD (eval_framework.tasks.base.Language attribute) NSE (eval_framework.tasks.base.Language attribute) NSF (eval_framework.tasks.base.Language attribute) NSG (eval_framework.tasks.base.Language attribute) NSH (eval_framework.tasks.base.Language attribute) NSI (eval_framework.tasks.base.Language attribute) NSK (eval_framework.tasks.base.Language attribute) NSL (eval_framework.tasks.base.Language attribute) NSM (eval_framework.tasks.base.Language attribute) NSN (eval_framework.tasks.base.Language attribute) NSO (eval_framework.tasks.base.Language attribute) NSP (eval_framework.tasks.base.Language attribute) NSQ (eval_framework.tasks.base.Language attribute) NSR (eval_framework.tasks.base.Language attribute) NSS (eval_framework.tasks.base.Language attribute) NST (eval_framework.tasks.base.Language attribute) NSU (eval_framework.tasks.base.Language attribute) NSV (eval_framework.tasks.base.Language attribute) NSW (eval_framework.tasks.base.Language attribute) NSX (eval_framework.tasks.base.Language attribute) NSY (eval_framework.tasks.base.Language attribute) NSZ (eval_framework.tasks.base.Language attribute) NTD (eval_framework.tasks.base.Language attribute) NTE (eval_framework.tasks.base.Language attribute) NTG (eval_framework.tasks.base.Language attribute) NTI (eval_framework.tasks.base.Language attribute) NTJ (eval_framework.tasks.base.Language attribute) NTK (eval_framework.tasks.base.Language attribute) NTM (eval_framework.tasks.base.Language attribute) NTO (eval_framework.tasks.base.Language attribute) NTP (eval_framework.tasks.base.Language attribute) NTR (eval_framework.tasks.base.Language attribute) NTS (eval_framework.tasks.base.Language attribute) NTU (eval_framework.tasks.base.Language attribute) NTW (eval_framework.tasks.base.Language attribute) NTX (eval_framework.tasks.base.Language attribute) NTY (eval_framework.tasks.base.Language attribute) NTZ (eval_framework.tasks.base.Language attribute) NUA (eval_framework.tasks.base.Language attribute) NUC (eval_framework.tasks.base.Language attribute) NUD (eval_framework.tasks.base.Language attribute) NUE (eval_framework.tasks.base.Language attribute) NUF (eval_framework.tasks.base.Language attribute) NUG (eval_framework.tasks.base.Language attribute) NUH (eval_framework.tasks.base.Language attribute) NUI (eval_framework.tasks.base.Language attribute) NUJ (eval_framework.tasks.base.Language attribute) NUK (eval_framework.tasks.base.Language attribute) NUL (eval_framework.tasks.base.Language attribute) NUM (eval_framework.tasks.base.Language attribute) num_fewshot (eval_framework.context.determined.TaskArgs attribute) (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) num_placeholders (eval_framework.metrics.completion.placeholder_checker.PlaceholderCheckerMetricContext attribute) num_samples (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) NUN (eval_framework.tasks.base.Language attribute) NUO (eval_framework.tasks.base.Language attribute) NUP (eval_framework.tasks.base.Language attribute) NUQ (eval_framework.tasks.base.Language attribute) NUR (eval_framework.tasks.base.Language attribute) NUS (eval_framework.tasks.base.Language attribute) NUT (eval_framework.tasks.base.Language attribute) NUU (eval_framework.tasks.base.Language attribute) NUV (eval_framework.tasks.base.Language attribute) NUW (eval_framework.tasks.base.Language attribute) NUX (eval_framework.tasks.base.Language attribute) NUY (eval_framework.tasks.base.Language attribute) NUZ (eval_framework.tasks.base.Language attribute) NVH (eval_framework.tasks.base.Language attribute) NVM (eval_framework.tasks.base.Language attribute) NVO (eval_framework.tasks.base.Language attribute) NWA (eval_framework.tasks.base.Language attribute) NWB (eval_framework.tasks.base.Language attribute) NWC (eval_framework.tasks.base.Language attribute) NWE (eval_framework.tasks.base.Language attribute) NWG (eval_framework.tasks.base.Language attribute) NWI (eval_framework.tasks.base.Language attribute) NWM (eval_framework.tasks.base.Language attribute) NWO (eval_framework.tasks.base.Language attribute) NWR (eval_framework.tasks.base.Language attribute) NWW (eval_framework.tasks.base.Language attribute) NWX (eval_framework.tasks.base.Language attribute) NWY (eval_framework.tasks.base.Language attribute) NXA (eval_framework.tasks.base.Language attribute) NXD (eval_framework.tasks.base.Language attribute) NXE (eval_framework.tasks.base.Language attribute) NXG (eval_framework.tasks.base.Language attribute) NXI (eval_framework.tasks.base.Language attribute) NXJ (eval_framework.tasks.base.Language attribute) NXK (eval_framework.tasks.base.Language attribute) NXL (eval_framework.tasks.base.Language attribute) NXM (eval_framework.tasks.base.Language attribute) NXN (eval_framework.tasks.base.Language attribute) NXO (eval_framework.tasks.base.Language attribute) NXQ (eval_framework.tasks.base.Language attribute) NXR (eval_framework.tasks.base.Language attribute) NXU (eval_framework.tasks.base.Language attribute) NXX (eval_framework.tasks.base.Language attribute) NYA (eval_framework.tasks.base.Language attribute) NYB (eval_framework.tasks.base.Language attribute) NYC (eval_framework.tasks.base.Language attribute) NYD (eval_framework.tasks.base.Language attribute) NYE (eval_framework.tasks.base.Language attribute) NYF (eval_framework.tasks.base.Language attribute) NYG (eval_framework.tasks.base.Language attribute) NYH (eval_framework.tasks.base.Language attribute) NYI (eval_framework.tasks.base.Language attribute) NYJ (eval_framework.tasks.base.Language attribute) NYK (eval_framework.tasks.base.Language attribute) NYL (eval_framework.tasks.base.Language attribute) NYM (eval_framework.tasks.base.Language attribute) NYN (eval_framework.tasks.base.Language attribute) NYO (eval_framework.tasks.base.Language attribute) NYP (eval_framework.tasks.base.Language attribute) NYQ (eval_framework.tasks.base.Language attribute) NYR (eval_framework.tasks.base.Language attribute) NYS (eval_framework.tasks.base.Language attribute) NYT (eval_framework.tasks.base.Language attribute) NYU (eval_framework.tasks.base.Language attribute) NYV (eval_framework.tasks.base.Language attribute) NYW (eval_framework.tasks.base.Language attribute) NYX (eval_framework.tasks.base.Language attribute) NYY (eval_framework.tasks.base.Language attribute) NZA (eval_framework.tasks.base.Language attribute) NZB (eval_framework.tasks.base.Language attribute) NZD (eval_framework.tasks.base.Language attribute) NZI (eval_framework.tasks.base.Language attribute) NZK (eval_framework.tasks.base.Language attribute) NZM (eval_framework.tasks.base.Language attribute) NZR (eval_framework.tasks.base.Language attribute) NZS (eval_framework.tasks.base.Language attribute) NZU (eval_framework.tasks.base.Language attribute) NZY (eval_framework.tasks.base.Language attribute) NZZ (eval_framework.tasks.base.Language attribute) O OAA (eval_framework.tasks.base.Language attribute) OAC (eval_framework.tasks.base.Language attribute) OAK (eval_framework.tasks.base.Language attribute) OAR (eval_framework.tasks.base.Language attribute) OAV (eval_framework.tasks.base.Language attribute) OBI (eval_framework.tasks.base.Language attribute) OBK (eval_framework.tasks.base.Language attribute) OBL (eval_framework.tasks.base.Language attribute) OBM (eval_framework.tasks.base.Language attribute) OBO (eval_framework.tasks.base.Language attribute) OBR (eval_framework.tasks.base.Language attribute) OBT (eval_framework.tasks.base.Language attribute) OBU (eval_framework.tasks.base.Language attribute) OCA (eval_framework.tasks.base.Language attribute) OCC (eval_framework.tasks.base.Language attribute) OCH (eval_framework.tasks.base.Language attribute) OCI (eval_framework.tasks.base.Language attribute) OCM (eval_framework.tasks.base.Language attribute) OCO (eval_framework.tasks.base.Language attribute) OCU (eval_framework.tasks.base.Language attribute) ODA (eval_framework.tasks.base.Language attribute) ODK (eval_framework.tasks.base.Language attribute) ODT (eval_framework.tasks.base.Language attribute) ODU (eval_framework.tasks.base.Language attribute) OFO (eval_framework.tasks.base.Language attribute) OFS (eval_framework.tasks.base.Language attribute) OFU (eval_framework.tasks.base.Language attribute) OGB (eval_framework.tasks.base.Language attribute) OGC (eval_framework.tasks.base.Language attribute) OGE (eval_framework.tasks.base.Language attribute) OGG (eval_framework.tasks.base.Language attribute) OGN (eval_framework.tasks.base.Language attribute) OGO (eval_framework.tasks.base.Language attribute) OGU (eval_framework.tasks.base.Language attribute) OHT (eval_framework.tasks.base.Language attribute) OHU (eval_framework.tasks.base.Language attribute) OIA (eval_framework.tasks.base.Language attribute) OIE (eval_framework.tasks.base.Language attribute) OIN (eval_framework.tasks.base.Language attribute) OJB (eval_framework.tasks.base.Language attribute) OJC (eval_framework.tasks.base.Language attribute) OJG (eval_framework.tasks.base.Language attribute) OJI (eval_framework.tasks.base.Language attribute) OJP (eval_framework.tasks.base.Language attribute) OJS (eval_framework.tasks.base.Language attribute) OJV (eval_framework.tasks.base.Language attribute) OJW (eval_framework.tasks.base.Language attribute) OKA (eval_framework.tasks.base.Language attribute) OKB (eval_framework.tasks.base.Language attribute) OKC (eval_framework.tasks.base.Language attribute) OKD (eval_framework.tasks.base.Language attribute) OKE (eval_framework.tasks.base.Language attribute) OKG (eval_framework.tasks.base.Language attribute) OKH (eval_framework.tasks.base.Language attribute) OKI (eval_framework.tasks.base.Language attribute) OKJ (eval_framework.tasks.base.Language attribute) OKK (eval_framework.tasks.base.Language attribute) OKL (eval_framework.tasks.base.Language attribute) OKM (eval_framework.tasks.base.Language attribute) OKN (eval_framework.tasks.base.Language attribute) OKO (eval_framework.tasks.base.Language attribute) OKR (eval_framework.tasks.base.Language attribute) OKS (eval_framework.tasks.base.Language attribute) OKU (eval_framework.tasks.base.Language attribute) OKV (eval_framework.tasks.base.Language attribute) OKX (eval_framework.tasks.base.Language attribute) OKZ (eval_framework.tasks.base.Language attribute) OLA (eval_framework.tasks.base.Language attribute) OLB (eval_framework.tasks.base.Language attribute) OLD (eval_framework.tasks.base.Language attribute) OLE (eval_framework.tasks.base.Language attribute) OLK (eval_framework.tasks.base.Language attribute) OLM (eval_framework.tasks.base.Language attribute) OLO (eval_framework.tasks.base.Language attribute) OLR (eval_framework.tasks.base.Language attribute) OLT (eval_framework.tasks.base.Language attribute) OLU (eval_framework.tasks.base.Language attribute) OMA (eval_framework.tasks.base.Language attribute) OMB (eval_framework.tasks.base.Language attribute) OMC (eval_framework.tasks.base.Language attribute) OME (eval_framework.tasks.base.Language attribute) OMG (eval_framework.tasks.base.Language attribute) OMI (eval_framework.tasks.base.Language attribute) OMK (eval_framework.tasks.base.Language attribute) OML (eval_framework.tasks.base.Language attribute) OMN (eval_framework.tasks.base.Language attribute) OMO (eval_framework.tasks.base.Language attribute) OMP (eval_framework.tasks.base.Language attribute) OMR (eval_framework.tasks.base.Language attribute) OMT (eval_framework.tasks.base.Language attribute) OMU (eval_framework.tasks.base.Language attribute) OMW (eval_framework.tasks.base.Language attribute) OMX (eval_framework.tasks.base.Language attribute) OMY (eval_framework.tasks.base.Language attribute) ONA (eval_framework.tasks.base.Language attribute) ONB (eval_framework.tasks.base.Language attribute) ONE (eval_framework.tasks.base.Language attribute) ONG (eval_framework.tasks.base.Language attribute) ONI (eval_framework.tasks.base.Language attribute) ONJ (eval_framework.tasks.base.Language attribute) ONK (eval_framework.tasks.base.Language attribute) ONN (eval_framework.tasks.base.Language attribute) ONO (eval_framework.tasks.base.Language attribute) ONP (eval_framework.tasks.base.Language attribute) ONR (eval_framework.tasks.base.Language attribute) ONS (eval_framework.tasks.base.Language attribute) ONT (eval_framework.tasks.base.Language attribute) ONU (eval_framework.tasks.base.Language attribute) ONW (eval_framework.tasks.base.Language attribute) ONX (eval_framework.tasks.base.Language attribute) OOD (eval_framework.tasks.base.Language attribute) OOG (eval_framework.tasks.base.Language attribute) OON (eval_framework.tasks.base.Language attribute) OOR (eval_framework.tasks.base.Language attribute) OOS (eval_framework.tasks.base.Language attribute) OPA (eval_framework.tasks.base.Language attribute) OPE (eval_framework.tasks.base.Language attribute) OpenAI_davinci_002 (class in eval_framework.llm.openai) OpenAI_gpt_4o_mini (class in eval_framework.llm.openai) OpenAI_gpt_4o_mini_with_ConcatFormatter (class in eval_framework.llm.openai) OpenAIEmbeddingModel (class in eval_framework.llm.openai) OpenAIModel (class in eval_framework.llm.openai) OPENBOOKQA (class in eval_framework.tasks.benchmarks.openbookqa) OPENBOOKQA_EVAL_HARNESS (class in eval_framework.tasks.benchmarks.openbookqa) OPENBOOKQA_IDK (class in eval_framework.tasks.benchmarks.openbookqa) OPK (eval_framework.tasks.base.Language attribute) OPM (eval_framework.tasks.base.Language attribute) OPO (eval_framework.tasks.base.Language attribute) OPT (eval_framework.tasks.base.Language attribute) OPY (eval_framework.tasks.base.Language attribute) ORA (eval_framework.tasks.base.Language attribute) ORC (eval_framework.tasks.base.Language attribute) order_answers_for_comparison() (in module eval_framework.metrics.llm.utils) ORE (eval_framework.tasks.base.Language attribute) ORG (eval_framework.tasks.base.Language attribute) ORH (eval_framework.tasks.base.Language attribute) ORI (eval_framework.tasks.base.Language attribute) ORK (eval_framework.tasks.base.Language attribute) ORM (eval_framework.tasks.base.Language attribute) ORN (eval_framework.tasks.base.Language attribute) ORO (eval_framework.tasks.base.Language attribute) ORR (eval_framework.tasks.base.Language attribute) ORS (eval_framework.tasks.base.Language attribute) ORT (eval_framework.tasks.base.Language attribute) ORU (eval_framework.tasks.base.Language attribute) ORV (eval_framework.tasks.base.Language attribute) ORW (eval_framework.tasks.base.Language attribute) ORX (eval_framework.tasks.base.Language attribute) ORY (eval_framework.tasks.base.Language attribute) ORZ (eval_framework.tasks.base.Language attribute) OSA (eval_framework.tasks.base.Language attribute) OSC (eval_framework.tasks.base.Language attribute) OSD (eval_framework.tasks.base.Language attribute) OSI (eval_framework.tasks.base.Language attribute) OSN (eval_framework.tasks.base.Language attribute) OSO (eval_framework.tasks.base.Language attribute) OSP (eval_framework.tasks.base.Language attribute) OSS (eval_framework.tasks.base.Language attribute) OST (eval_framework.tasks.base.Language attribute) OSU (eval_framework.tasks.base.Language attribute) OSX (eval_framework.tasks.base.Language attribute) OTA (eval_framework.tasks.base.Language attribute) OTB (eval_framework.tasks.base.Language attribute) OTD (eval_framework.tasks.base.Language attribute) OTE (eval_framework.tasks.base.Language attribute) OTI (eval_framework.tasks.base.Language attribute) OTK (eval_framework.tasks.base.Language attribute) OTL (eval_framework.tasks.base.Language attribute) OTM (eval_framework.tasks.base.Language attribute) OTN (eval_framework.tasks.base.Language attribute) OTQ (eval_framework.tasks.base.Language attribute) OTR (eval_framework.tasks.base.Language attribute) OTS (eval_framework.tasks.base.Language attribute) OTT (eval_framework.tasks.base.Language attribute) OTU (eval_framework.tasks.base.Language attribute) OTW (eval_framework.tasks.base.Language attribute) OTX (eval_framework.tasks.base.Language attribute) OTY (eval_framework.tasks.base.Language attribute) OTZ (eval_framework.tasks.base.Language attribute) OUA (eval_framework.tasks.base.Language attribute) OUB (eval_framework.tasks.base.Language attribute) OUE (eval_framework.tasks.base.Language attribute) OUI (eval_framework.tasks.base.Language attribute) OUM (eval_framework.tasks.base.Language attribute) OUN (eval_framework.tasks.base.Language attribute) output (eval_framework.tasks.utils.ExecutionResult attribute), [1] output_dir (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) output_parse_fn (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext attribute) output_type (eval_framework.metrics.completion.struct_eval_metrics.RenderableStructMetricContext attribute) (eval_framework.metrics.completion.struct_eval_metrics.StructMetricContext attribute) OVD (eval_framework.tasks.base.Language attribute) OWI (eval_framework.tasks.base.Language attribute) OWL (eval_framework.tasks.base.Language attribute) OYB (eval_framework.tasks.base.Language attribute) OYD (eval_framework.tasks.base.Language attribute) OYM (eval_framework.tasks.base.Language attribute) OYY (eval_framework.tasks.base.Language attribute) OZM (eval_framework.tasks.base.Language attribute) P PAB (eval_framework.tasks.base.Language attribute) PAC (eval_framework.tasks.base.Language attribute) package_downloads (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) PAD (eval_framework.tasks.base.Language attribute) PAE (eval_framework.tasks.base.Language attribute) PAF (eval_framework.tasks.base.Language attribute) PAG (eval_framework.tasks.base.Language attribute) PAH (eval_framework.tasks.base.Language attribute) PAI (eval_framework.tasks.base.Language attribute) PAJ (eval_framework.tasks.base.Language attribute) PAK (eval_framework.tasks.base.Language attribute) PAL (eval_framework.tasks.base.Language attribute) PAM (eval_framework.tasks.base.Language attribute) PAN (eval_framework.tasks.base.Language attribute) PAO (eval_framework.tasks.base.Language attribute) PAP (eval_framework.tasks.base.Language attribute) PAQ (eval_framework.tasks.base.Language attribute) PAR (eval_framework.tasks.base.Language attribute) paragraph_count (eval_framework.metrics.completion.text_counter.ParagraphCounterMetricContext attribute) ParagraphCounter (class in eval_framework.metrics.completion.text_counter) ParagraphCounterMetricContext (class in eval_framework.metrics.completion.text_counter) PARAGRAPHS (eval_framework.metrics.completion.length_control.LengthRequirementUnit attribute) parse_args() (in module eval_framework.run) PAS (eval_framework.tasks.base.Language attribute) PAT (eval_framework.tasks.base.Language attribute) path_exists() (in module eval_framework.metrics.completion.struct_eval_metrics) paths (eval_framework.metrics.completion.struct_eval_metrics.StructMetricContext attribute) PAU (eval_framework.tasks.base.Language attribute) PAV (eval_framework.tasks.base.Language attribute) PAW (eval_framework.tasks.base.Language attribute) PAWSX (class in eval_framework.tasks.benchmarks.pawsx) PAX (eval_framework.tasks.base.Language attribute) PAY (eval_framework.tasks.base.Language attribute) PAZ (eval_framework.tasks.base.Language attribute) PBB (eval_framework.tasks.base.Language attribute) PBC (eval_framework.tasks.base.Language attribute) PBE (eval_framework.tasks.base.Language attribute) PBF (eval_framework.tasks.base.Language attribute) PBG (eval_framework.tasks.base.Language attribute) PBH (eval_framework.tasks.base.Language attribute) PBI (eval_framework.tasks.base.Language attribute) PBL (eval_framework.tasks.base.Language attribute) PBM (eval_framework.tasks.base.Language attribute) PBN (eval_framework.tasks.base.Language attribute) PBO (eval_framework.tasks.base.Language attribute) PBP (eval_framework.tasks.base.Language attribute) PBR (eval_framework.tasks.base.Language attribute) PBS (eval_framework.tasks.base.Language attribute) PBT (eval_framework.tasks.base.Language attribute) PBU (eval_framework.tasks.base.Language attribute) PBV (eval_framework.tasks.base.Language attribute) PBY (eval_framework.tasks.base.Language attribute) PBZ (eval_framework.tasks.base.Language attribute) PCA (eval_framework.tasks.base.Language attribute) PCB (eval_framework.tasks.base.Language attribute) PCC (eval_framework.tasks.base.Language attribute) PCD (eval_framework.tasks.base.Language attribute) PCE (eval_framework.tasks.base.Language attribute) PCF (eval_framework.tasks.base.Language attribute) PCG (eval_framework.tasks.base.Language attribute) PCH (eval_framework.tasks.base.Language attribute) PCI (eval_framework.tasks.base.Language attribute) PCJ (eval_framework.tasks.base.Language attribute) PCK (eval_framework.tasks.base.Language attribute) PCL (eval_framework.tasks.base.Language attribute) PCM (eval_framework.tasks.base.Language attribute) PCN (eval_framework.tasks.base.Language attribute) PCP (eval_framework.tasks.base.Language attribute) PCR (eval_framework.tasks.base.Language attribute) PCW (eval_framework.tasks.base.Language attribute) PDA (eval_framework.tasks.base.Language attribute) PDC (eval_framework.tasks.base.Language attribute) PDI (eval_framework.tasks.base.Language attribute) PDN (eval_framework.tasks.base.Language attribute) PDO (eval_framework.tasks.base.Language attribute) PDT (eval_framework.tasks.base.Language attribute) PDU (eval_framework.tasks.base.Language attribute) PEA (eval_framework.tasks.base.Language attribute) PEB (eval_framework.tasks.base.Language attribute) PEC (eval_framework.tasks.base.Language attribute) PED (eval_framework.tasks.base.Language attribute) PEE (eval_framework.tasks.base.Language attribute) PEF (eval_framework.tasks.base.Language attribute) PEG (eval_framework.tasks.base.Language attribute) PEH (eval_framework.tasks.base.Language attribute) PEI (eval_framework.tasks.base.Language attribute) PEJ (eval_framework.tasks.base.Language attribute) PEK (eval_framework.tasks.base.Language attribute) PEL (eval_framework.tasks.base.Language attribute) PEM (eval_framework.tasks.base.Language attribute) PEN (eval_framework.tasks.base.Language attribute) PEO (eval_framework.tasks.base.Language attribute) PEP (eval_framework.tasks.base.Language attribute) PEQ (eval_framework.tasks.base.Language attribute) PERMUTE (eval_framework.tasks.perturbation.PerturbationType attribute) permute_chars_in_string() (eval_framework.tasks.utils.HatPaperEditor method) perturbation_config (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) PERTURBATION_UNMODIFIABLE_WORDS (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA_COT attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.mmlu.FullTextMMLU attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.wmt.WMT_INSTRUCT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_GOV_REPORT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_MUSIQUE attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_NARRATIVEQA attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QASPER attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QMSUM attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SPACE_DIGEST attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SQUALITY attribute) PerturbationConfig (class in eval_framework.tasks.perturbation) PerturbationType (class in eval_framework.tasks.perturbation) PES (eval_framework.tasks.base.Language attribute) PEV (eval_framework.tasks.base.Language attribute) PEX (eval_framework.tasks.base.Language attribute) PEY (eval_framework.tasks.base.Language attribute) PEZ (eval_framework.tasks.base.Language attribute) PFA (eval_framework.tasks.base.Language attribute) PFE (eval_framework.tasks.base.Language attribute) PFL (eval_framework.tasks.base.Language attribute) PGA (eval_framework.tasks.base.Language attribute) PGD (eval_framework.tasks.base.Language attribute) PGG (eval_framework.tasks.base.Language attribute) PGI (eval_framework.tasks.base.Language attribute) PGK (eval_framework.tasks.base.Language attribute) PGL (eval_framework.tasks.base.Language attribute) PGN (eval_framework.tasks.base.Language attribute) PGS (eval_framework.tasks.base.Language attribute) PGU (eval_framework.tasks.base.Language attribute) PGY (eval_framework.tasks.base.Language attribute) PGZ (eval_framework.tasks.base.Language attribute) PHA (eval_framework.tasks.base.Language attribute) PHD (eval_framework.tasks.base.Language attribute) PHG (eval_framework.tasks.base.Language attribute) PHH (eval_framework.tasks.base.Language attribute) PHJ (eval_framework.tasks.base.Language attribute) PHK (eval_framework.tasks.base.Language attribute) PHL (eval_framework.tasks.base.Language attribute) PHM (eval_framework.tasks.base.Language attribute) PHN (eval_framework.tasks.base.Language attribute) PHO (eval_framework.tasks.base.Language attribute) PHQ (eval_framework.tasks.base.Language attribute) PHR (eval_framework.tasks.base.Language attribute) PHT (eval_framework.tasks.base.Language attribute) PHU (eval_framework.tasks.base.Language attribute) PHV (eval_framework.tasks.base.Language attribute) PHW (eval_framework.tasks.base.Language attribute) PIA (eval_framework.tasks.base.Language attribute) PIB (eval_framework.tasks.base.Language attribute) PIC (eval_framework.tasks.base.Language attribute) PID (eval_framework.tasks.base.Language attribute) PIE (eval_framework.tasks.base.Language attribute) PIF (eval_framework.tasks.base.Language attribute) PIG (eval_framework.tasks.base.Language attribute) PIH (eval_framework.tasks.base.Language attribute) PII (eval_framework.tasks.base.Language attribute) PIJ (eval_framework.tasks.base.Language attribute) PIL (eval_framework.tasks.base.Language attribute) PIM (eval_framework.tasks.base.Language attribute) PIN (eval_framework.tasks.base.Language attribute) PIO (eval_framework.tasks.base.Language attribute) PIP (eval_framework.tasks.base.Language attribute) PIQA (class in eval_framework.tasks.benchmarks.piqa) PIQA_IDK (class in eval_framework.tasks.benchmarks.piqa) PIR (eval_framework.tasks.base.Language attribute) PIS (eval_framework.tasks.base.Language attribute) PIT (eval_framework.tasks.base.Language attribute) PIU (eval_framework.tasks.base.Language attribute) PIV (eval_framework.tasks.base.Language attribute) PIW (eval_framework.tasks.base.Language attribute) PIX (eval_framework.tasks.base.Language attribute) PIY (eval_framework.tasks.base.Language attribute) PIZ (eval_framework.tasks.base.Language attribute) PJT (eval_framework.tasks.base.Language attribute) PKA (eval_framework.tasks.base.Language attribute) PKB (eval_framework.tasks.base.Language attribute) PKC (eval_framework.tasks.base.Language attribute) PKG (eval_framework.tasks.base.Language attribute) PKH (eval_framework.tasks.base.Language attribute) PKN (eval_framework.tasks.base.Language attribute) PKO (eval_framework.tasks.base.Language attribute) PKP (eval_framework.tasks.base.Language attribute) PKR (eval_framework.tasks.base.Language attribute) PKS (eval_framework.tasks.base.Language attribute) PKT (eval_framework.tasks.base.Language attribute) PKU (eval_framework.tasks.base.Language attribute) PLA (eval_framework.tasks.base.Language attribute) PlaceholderChecker (class in eval_framework.metrics.completion.placeholder_checker) PlaceholderCheckerMetricContext (class in eval_framework.metrics.completion.placeholder_checker) PLB (eval_framework.tasks.base.Language attribute) PLC (eval_framework.tasks.base.Language attribute) PLD (eval_framework.tasks.base.Language attribute) PLE (eval_framework.tasks.base.Language attribute) PLG (eval_framework.tasks.base.Language attribute) PLH (eval_framework.tasks.base.Language attribute) PLI (eval_framework.tasks.base.Language attribute) PLJ (eval_framework.tasks.base.Language attribute) PLK (eval_framework.tasks.base.Language attribute) PLL (eval_framework.tasks.base.Language attribute) PLM (eval_framework.tasks.base.Language attribute) PLN (eval_framework.tasks.base.Language attribute) PLO (eval_framework.tasks.base.Language attribute) PLP (eval_framework.tasks.base.Language attribute) PLQ (eval_framework.tasks.base.Language attribute) PLR (eval_framework.tasks.base.Language attribute) PLS (eval_framework.tasks.base.Language attribute) PLT (eval_framework.tasks.base.Language attribute) PLU (eval_framework.tasks.base.Language attribute) PLV (eval_framework.tasks.base.Language attribute) PLW (eval_framework.tasks.base.Language attribute) PLY (eval_framework.tasks.base.Language attribute) PLZ (eval_framework.tasks.base.Language attribute) PMA (eval_framework.tasks.base.Language attribute) PMB (eval_framework.tasks.base.Language attribute) PMC (eval_framework.tasks.base.Language attribute) PMD (eval_framework.tasks.base.Language attribute) PME (eval_framework.tasks.base.Language attribute) PMF (eval_framework.tasks.base.Language attribute) PMH (eval_framework.tasks.base.Language attribute) PMI (eval_framework.tasks.base.Language attribute) PMJ (eval_framework.tasks.base.Language attribute) PMK (eval_framework.tasks.base.Language attribute) PML (eval_framework.tasks.base.Language attribute) PMM (eval_framework.tasks.base.Language attribute) PMN (eval_framework.tasks.base.Language attribute) PMO (eval_framework.tasks.base.Language attribute) PMQ (eval_framework.tasks.base.Language attribute) PMR (eval_framework.tasks.base.Language attribute) PMS (eval_framework.tasks.base.Language attribute) PMT (eval_framework.tasks.base.Language attribute) PMU (eval_framework.tasks.base.Language attribute) PMW (eval_framework.tasks.base.Language attribute) PMX (eval_framework.tasks.base.Language attribute) PMY (eval_framework.tasks.base.Language attribute) PMZ (eval_framework.tasks.base.Language attribute) PNA (eval_framework.tasks.base.Language attribute) PNB (eval_framework.tasks.base.Language attribute) PNC (eval_framework.tasks.base.Language attribute) PND (eval_framework.tasks.base.Language attribute) PNE (eval_framework.tasks.base.Language attribute) PNG (eval_framework.tasks.base.Language attribute) PNH (eval_framework.tasks.base.Language attribute) PNI (eval_framework.tasks.base.Language attribute) PNJ (eval_framework.tasks.base.Language attribute) PNK (eval_framework.tasks.base.Language attribute) PNL (eval_framework.tasks.base.Language attribute) PNM (eval_framework.tasks.base.Language attribute) PNN (eval_framework.tasks.base.Language attribute) PNO (eval_framework.tasks.base.Language attribute) PNP (eval_framework.tasks.base.Language attribute) PNQ (eval_framework.tasks.base.Language attribute) PNR (eval_framework.tasks.base.Language attribute) PNS (eval_framework.tasks.base.Language attribute) PNT (eval_framework.tasks.base.Language attribute) PNU (eval_framework.tasks.base.Language attribute) PNV (eval_framework.tasks.base.Language attribute) PNW (eval_framework.tasks.base.Language attribute) PNX (eval_framework.tasks.base.Language attribute) PNY (eval_framework.tasks.base.Language attribute) PNZ (eval_framework.tasks.base.Language attribute) POA (eval_framework.tasks.base.Language attribute) POB (eval_framework.tasks.base.Language attribute) POC (eval_framework.tasks.base.Language attribute) POD (eval_framework.tasks.base.Language attribute) POE (eval_framework.tasks.base.Language attribute) POF (eval_framework.tasks.base.Language attribute) POG (eval_framework.tasks.base.Language attribute) POH (eval_framework.tasks.base.Language attribute) POI (eval_framework.tasks.base.Language attribute) POJ (eval_framework.tasks.base.Language attribute) POK (eval_framework.tasks.base.Language attribute) POL (eval_framework.tasks.base.Language attribute) POM (eval_framework.tasks.base.Language attribute) PON (eval_framework.tasks.base.Language attribute) POO (eval_framework.tasks.base.Language attribute) POP (eval_framework.tasks.base.Language attribute) POQ (eval_framework.tasks.base.Language attribute) POR (eval_framework.tasks.base.Language attribute) POS (eval_framework.tasks.base.Language attribute) possible_completions (eval_framework.tasks.base.Sample attribute) post_process_completion() (eval_framework.llm.base.BaseLLM method) post_process_generated_completion() (eval_framework.tasks.base.BaseTask method) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench method) (eval_framework.tasks.benchmarks.duc.DUC method) (eval_framework.tasks.benchmarks.flores200.Flores200 method) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus method) (eval_framework.tasks.benchmarks.gpqa.GPQA_COT method) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness method) (eval_framework.tasks.benchmarks.humaneval.HumanEval method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeRun method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnDia method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnQA method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_MathFind method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveKV2 method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveNumber method) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrievePassKey1 method) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning method) (eval_framework.tasks.benchmarks.math_reasoning.MATH method) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 method) (eval_framework.tasks.benchmarks.math_reasoning.MATHReasoning method) (eval_framework.tasks.benchmarks.mbpp.MBPP method) (eval_framework.tasks.benchmarks.mbpp.MBPP_PROMPT_WITHOUT_TESTS method) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT method) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT method) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT method) (eval_framework.tasks.benchmarks.pawsx.PAWSX method) (eval_framework.tasks.benchmarks.struct_eval.StructEval method) (eval_framework.tasks.benchmarks.tablebench.TableBench method) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA method) (eval_framework.tasks.benchmarks.wmt.WMT method) (eval_framework.tasks.benchmarks.wmt.WMT_INSTRUCT method) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SPACE_DIGEST method) postgres (eval_framework.metrics.llm.llm_judge_sql.SqlDialects attribute) POT (eval_framework.tasks.base.Language attribute) POU (eval_framework.tasks.base.Language attribute) POV (eval_framework.tasks.base.Language attribute) POW (eval_framework.tasks.base.Language attribute) POX (eval_framework.tasks.base.Language attribute) POY (eval_framework.tasks.base.Language attribute) PPA (eval_framework.tasks.base.Language attribute) PPE (eval_framework.tasks.base.Language attribute) PPI (eval_framework.tasks.base.Language attribute) PPK (eval_framework.tasks.base.Language attribute) PPL (eval_framework.tasks.base.Language attribute) PPM (eval_framework.tasks.base.Language attribute) PPN (eval_framework.tasks.base.Language attribute) PPO (eval_framework.tasks.base.Language attribute) PPP (eval_framework.tasks.base.Language attribute) PPQ (eval_framework.tasks.base.Language attribute) PPR (eval_framework.tasks.base.Language attribute) PPS (eval_framework.tasks.base.Language attribute) PPT (eval_framework.tasks.base.Language attribute) PPU (eval_framework.tasks.base.Language attribute) PPV (eval_framework.tasks.base.Language attribute) PQA (eval_framework.tasks.base.Language attribute) PQM (eval_framework.tasks.base.Language attribute) PRB (eval_framework.tasks.base.Language attribute) PRC (eval_framework.tasks.base.Language attribute) PRD (eval_framework.tasks.base.Language attribute) PRE (eval_framework.tasks.base.Language attribute) PRF (eval_framework.tasks.base.Language attribute) PRG (eval_framework.tasks.base.Language attribute) PRH (eval_framework.tasks.base.Language attribute) PRI (eval_framework.tasks.base.Language attribute) PRK (eval_framework.tasks.base.Language attribute) PRL (eval_framework.tasks.base.Language attribute) PRM (eval_framework.tasks.base.Language attribute) PRN (eval_framework.tasks.base.Language attribute) PRO (eval_framework.tasks.base.Language attribute) probability (eval_framework.tasks.perturbation.PerturbationConfig attribute) ProbabilityMass (class in eval_framework.metrics.loglikelihood.probability_mass) ProbabilityMassNorm (class in eval_framework.metrics.loglikelihood.probability_mass) prompt (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEvalMetricContext attribute) prompt_text (eval_framework.metrics.llm.llm_judge_mtbench_pair.PromptToJudge attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.PromptToJudge attribute) PromptToJudge (class in eval_framework.metrics.llm.llm_judge_mtbench_pair) (class in eval_framework.metrics.llm.llm_judge_mtbench_single) PRP (eval_framework.tasks.base.Language attribute) PRQ (eval_framework.tasks.base.Language attribute) PRR (eval_framework.tasks.base.Language attribute) PRS (eval_framework.tasks.base.Language attribute) PRT (eval_framework.tasks.base.Language attribute) PRU (eval_framework.tasks.base.Language attribute) PRV (eval_framework.tasks.base.Language attribute) PRW (eval_framework.tasks.base.Language attribute) PRX (eval_framework.tasks.base.Language attribute) PRY (eval_framework.tasks.base.Language attribute) PRZ (eval_framework.tasks.base.Language attribute) PSA (eval_framework.tasks.base.Language attribute) PSC (eval_framework.tasks.base.Language attribute) PSD (eval_framework.tasks.base.Language attribute) PSE (eval_framework.tasks.base.Language attribute) PSG (eval_framework.tasks.base.Language attribute) PSH (eval_framework.tasks.base.Language attribute) PSI (eval_framework.tasks.base.Language attribute) PSL (eval_framework.tasks.base.Language attribute) PSM (eval_framework.tasks.base.Language attribute) PSN (eval_framework.tasks.base.Language attribute) PSO (eval_framework.tasks.base.Language attribute) PSP (eval_framework.tasks.base.Language attribute) PSQ (eval_framework.tasks.base.Language attribute) PSR (eval_framework.tasks.base.Language attribute) PSS (eval_framework.tasks.base.Language attribute) PST (eval_framework.tasks.base.Language attribute) PSU (eval_framework.tasks.base.Language attribute) PSW (eval_framework.tasks.base.Language attribute) PSY (eval_framework.tasks.base.Language attribute) PTA (eval_framework.tasks.base.Language attribute) PTH (eval_framework.tasks.base.Language attribute) PTI (eval_framework.tasks.base.Language attribute) PTN (eval_framework.tasks.base.Language attribute) PTO (eval_framework.tasks.base.Language attribute) PTP (eval_framework.tasks.base.Language attribute) PTQ (eval_framework.tasks.base.Language attribute) PTR (eval_framework.tasks.base.Language attribute) PTT (eval_framework.tasks.base.Language attribute) PTU (eval_framework.tasks.base.Language attribute) PTV (eval_framework.tasks.base.Language attribute) PTW (eval_framework.tasks.base.Language attribute) PTY (eval_framework.tasks.base.Language attribute) PUA (eval_framework.tasks.base.Language attribute) PUB (eval_framework.tasks.base.Language attribute) PUC (eval_framework.tasks.base.Language attribute) PUD (eval_framework.tasks.base.Language attribute) PUE (eval_framework.tasks.base.Language attribute) PUF (eval_framework.tasks.base.Language attribute) PUG (eval_framework.tasks.base.Language attribute) PUI (eval_framework.tasks.base.Language attribute) PUJ (eval_framework.tasks.base.Language attribute) PUK (eval_framework.tasks.base.Language attribute) PUM (eval_framework.tasks.base.Language attribute) PUN (eval_framework.tasks.base.Language attribute) PUO (eval_framework.tasks.base.Language attribute) PUP (eval_framework.tasks.base.Language attribute) PUQ (eval_framework.tasks.base.Language attribute) PUR (eval_framework.tasks.base.Language attribute) PUS (eval_framework.tasks.base.Language attribute) PUT (eval_framework.tasks.base.Language attribute) PUU (eval_framework.tasks.base.Language attribute) PUW (eval_framework.tasks.base.Language attribute) PUX (eval_framework.tasks.base.Language attribute) PUY (eval_framework.tasks.base.Language attribute) PUZ (eval_framework.tasks.base.Language attribute) PWA (eval_framework.tasks.base.Language attribute) PWB (eval_framework.tasks.base.Language attribute) PWG (eval_framework.tasks.base.Language attribute) PWI (eval_framework.tasks.base.Language attribute) PWM (eval_framework.tasks.base.Language attribute) PWN (eval_framework.tasks.base.Language attribute) PWO (eval_framework.tasks.base.Language attribute) PWR (eval_framework.tasks.base.Language attribute) PWW (eval_framework.tasks.base.Language attribute) PXM (eval_framework.tasks.base.Language attribute) PYE (eval_framework.tasks.base.Language attribute) PYM (eval_framework.tasks.base.Language attribute) PYN (eval_framework.tasks.base.Language attribute) PYS (eval_framework.tasks.base.Language attribute) Pythia410m (class in eval_framework.llm.huggingface) PYU (eval_framework.tasks.base.Language attribute) PYX (eval_framework.tasks.base.Language attribute) PYY (eval_framework.tasks.base.Language attribute) PZE (eval_framework.tasks.base.Language attribute) PZH (eval_framework.tasks.base.Language attribute) PZN (eval_framework.tasks.base.Language attribute) Q QUA (eval_framework.tasks.base.Language attribute) QUALITY (class in eval_framework.tasks.benchmarks.quality) QUB (eval_framework.tasks.base.Language attribute) QUC (eval_framework.tasks.base.Language attribute) QUD (eval_framework.tasks.base.Language attribute) QUE (eval_framework.tasks.base.Language attribute) query_error (eval_framework.metrics.llm.llm_judge_sql.SqlValidationResult attribute) QUERY_TEMPLATE (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) QUF (eval_framework.tasks.base.Language attribute) QUG (eval_framework.tasks.base.Language attribute) QUH (eval_framework.tasks.base.Language attribute) QUI (eval_framework.tasks.base.Language attribute) QUJ (eval_framework.tasks.base.Language attribute) QUK (eval_framework.tasks.base.Language attribute) QUL (eval_framework.tasks.base.Language attribute) QUM (eval_framework.tasks.base.Language attribute) QUN (eval_framework.tasks.base.Language attribute) QUP (eval_framework.tasks.base.Language attribute) QUQ (eval_framework.tasks.base.Language attribute) QUR (eval_framework.tasks.base.Language attribute) QUS (eval_framework.tasks.base.Language attribute) QUT (eval_framework.tasks.base.Language attribute) QUU (eval_framework.tasks.base.Language attribute) QUV (eval_framework.tasks.base.Language attribute) QUW (eval_framework.tasks.base.Language attribute) QUX (eval_framework.tasks.base.Language attribute) QUY (eval_framework.tasks.base.Language attribute) QUZ (eval_framework.tasks.base.Language attribute) QVA (eval_framework.tasks.base.Language attribute) QVC (eval_framework.tasks.base.Language attribute) QVE (eval_framework.tasks.base.Language attribute) QVH (eval_framework.tasks.base.Language attribute) QVI (eval_framework.tasks.base.Language attribute) QVJ (eval_framework.tasks.base.Language attribute) QVL (eval_framework.tasks.base.Language attribute) QVM (eval_framework.tasks.base.Language attribute) QVN (eval_framework.tasks.base.Language attribute) QVO (eval_framework.tasks.base.Language attribute) QVP (eval_framework.tasks.base.Language attribute) QVS (eval_framework.tasks.base.Language attribute) QVW (eval_framework.tasks.base.Language attribute) QVY (eval_framework.tasks.base.Language attribute) QVZ (eval_framework.tasks.base.Language attribute) QWA (eval_framework.tasks.base.Language attribute) QWC (eval_framework.tasks.base.Language attribute) Qwen3_0_6B (class in eval_framework.llm.huggingface) Qwen3_0_6B_VLLM (class in eval_framework.llm.vllm) Qwen3_0_6B_VLLM_No_Thinking (class in eval_framework.llm.vllm) QWH (eval_framework.tasks.base.Language attribute) QWM (eval_framework.tasks.base.Language attribute) QWS (eval_framework.tasks.base.Language attribute) QWT (eval_framework.tasks.base.Language attribute) QXA (eval_framework.tasks.base.Language attribute) QXC (eval_framework.tasks.base.Language attribute) QXH (eval_framework.tasks.base.Language attribute) QXI (eval_framework.tasks.base.Language attribute) QXL (eval_framework.tasks.base.Language attribute) QXN (eval_framework.tasks.base.Language attribute) QXO (eval_framework.tasks.base.Language attribute) QXP (eval_framework.tasks.base.Language attribute) QXQ (eval_framework.tasks.base.Language attribute) QXR (eval_framework.tasks.base.Language attribute) QXS (eval_framework.tasks.base.Language attribute) QXT (eval_framework.tasks.base.Language attribute) QXU (eval_framework.tasks.base.Language attribute) QXW (eval_framework.tasks.base.Language attribute) QYA (eval_framework.tasks.base.Language attribute) QYP (eval_framework.tasks.base.Language attribute) R RAA (eval_framework.tasks.base.Language attribute) RAB (eval_framework.tasks.base.Language attribute) RAC (eval_framework.tasks.base.Language attribute) RAD (eval_framework.tasks.base.Language attribute) RAE (eval_framework.tasks.base.Language attribute) RAF (eval_framework.tasks.base.Language attribute) RAG (eval_framework.tasks.base.Language attribute) RAH (eval_framework.tasks.base.Language attribute) RAI (eval_framework.tasks.base.Language attribute) raise_errors() (in module eval_framework.tasks.utils) RAJ (eval_framework.tasks.base.Language attribute) RAK (eval_framework.tasks.base.Language attribute) RAL (eval_framework.tasks.base.Language attribute) RAM (eval_framework.tasks.base.Language attribute) RAN (eval_framework.tasks.base.Language attribute) randomize_judge_order (eval_framework.tasks.eval_config.EvalConfig attribute) RAO (eval_framework.tasks.base.Language attribute) RAP (eval_framework.tasks.base.Language attribute) RAQ (eval_framework.tasks.base.Language attribute) RAR (eval_framework.tasks.base.Language attribute) RAS (eval_framework.tasks.base.Language attribute) RAT (eval_framework.tasks.base.Language attribute) RAU (eval_framework.tasks.base.Language attribute) RAV (eval_framework.tasks.base.Language attribute) RAW (eval_framework.tasks.base.Language attribute) RAX (eval_framework.tasks.base.Language attribute) RAY (eval_framework.tasks.base.Language attribute) RAZ (eval_framework.tasks.base.Language attribute) RBB (eval_framework.tasks.base.Language attribute) RBK (eval_framework.tasks.base.Language attribute) RBL (eval_framework.tasks.base.Language attribute) RBP (eval_framework.tasks.base.Language attribute) RCF (eval_framework.tasks.base.Language attribute) RDB (eval_framework.tasks.base.Language attribute) REA (eval_framework.tasks.base.Language attribute) RealtimeCodeExectionContext (class in eval_framework.metrics.completion.code_execution_pass_at_one) REB (eval_framework.tasks.base.Language attribute) REE (eval_framework.tasks.base.Language attribute) reference (eval_framework.metrics.llm.llm_judge_mtbench_pair.MTBenchJudgePairMetricContext attribute) (eval_framework.metrics.llm.llm_judge_mtbench_single.MTBenchJudgeSingleMetricContext attribute) REG (eval_framework.tasks.base.Language attribute) register_all_tasks() (in module eval_framework.tasks.task_names) register_artifact_upload_function() (in module eval_framework.result_processors.wandb_uploader) register_lazy_task() (in module eval_framework.tasks.registry) register_task() (in module eval_framework.tasks.registry) registered_task_names() (in module eval_framework.tasks.registry) registered_tasks_iter() (in module eval_framework.tasks.registry) Registry (class in eval_framework.tasks.registry) REI (eval_framework.tasks.base.Language attribute) REJ (eval_framework.tasks.base.Language attribute) REL (eval_framework.tasks.base.Language attribute) REM (eval_framework.tasks.base.Language attribute) remove_comments() (in module eval_framework.metrics.completion.json_format) REMOVED_EXPRESSIONS_FORMAT (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) REMOVED_EXPRESSIONS_UNITS (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) REN (eval_framework.tasks.base.Language attribute) RenderableStructEval (class in eval_framework.tasks.benchmarks.struct_eval) RenderableStructMetric (class in eval_framework.metrics.completion.struct_eval_metrics) RenderableStructMetricContext (class in eval_framework.metrics.completion.struct_eval_metrics) repeat_samples() (in module eval_framework.response_generator) RepeatedTokenSequenceCriteria (class in eval_framework.llm.huggingface) repeats (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) REPLACE (eval_framework.tasks.perturbation.PerturbationType attribute) replace_chars_in_string() (eval_framework.tasks.utils.HatPaperEditor method) RER (eval_framework.tasks.base.Language attribute) RES (eval_framework.tasks.base.Language attribute) response (eval_framework.result_processors.base.Result attribute) RESPONSE_TYPE (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA_COT attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBenchCompletion attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBenchLoglikelihood attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATHReasoning attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_COMPLETION attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) ResponseGenerator (class in eval_framework.response_generator) ResponseToOriginalBLEU (class in eval_framework.metrics.completion.bleu) ResponseToOriginalLengthRatio (class in eval_framework.metrics.completion.text_counter) ResponseType (class in eval_framework.tasks.base) Result (class in eval_framework.result_processors.base) ResultProcessor (class in eval_framework.result_processors.base) results (eval_framework.metrics.llm.llm_judge_sql.SqlValidationResult attribute) results_equal (eval_framework.metrics.llm.llm_judge_sql.SqlOutputComparison attribute) ResultsFileProcessor (class in eval_framework.result_processors.result_processor) ResultsUploader (class in eval_framework.result_processors.base) RET (eval_framework.tasks.base.Language attribute) REY (eval_framework.tasks.base.Language attribute) RGA (eval_framework.tasks.base.Language attribute) RGE (eval_framework.tasks.base.Language attribute) RGK (eval_framework.tasks.base.Language attribute) RGN (eval_framework.tasks.base.Language attribute) RGR (eval_framework.tasks.base.Language attribute) RGS (eval_framework.tasks.base.Language attribute) RGU (eval_framework.tasks.base.Language attribute) RHG (eval_framework.tasks.base.Language attribute) RHP (eval_framework.tasks.base.Language attribute) RIA (eval_framework.tasks.base.Language attribute) RIB (eval_framework.tasks.base.Language attribute) RIE (eval_framework.tasks.base.Language attribute) RIF (eval_framework.tasks.base.Language attribute) RIL (eval_framework.tasks.base.Language attribute) RIM (eval_framework.tasks.base.Language attribute) RIN (eval_framework.tasks.base.Language attribute) RIR (eval_framework.tasks.base.Language attribute) RIT (eval_framework.tasks.base.Language attribute) RIU (eval_framework.tasks.base.Language attribute) RJB (eval_framework.tasks.base.Language attribute) RJG (eval_framework.tasks.base.Language attribute) RJI (eval_framework.tasks.base.Language attribute) RJS (eval_framework.tasks.base.Language attribute) RKA (eval_framework.tasks.base.Language attribute) RKB (eval_framework.tasks.base.Language attribute) RKH (eval_framework.tasks.base.Language attribute) RKI (eval_framework.tasks.base.Language attribute) RKM (eval_framework.tasks.base.Language attribute) RKT (eval_framework.tasks.base.Language attribute) RKW (eval_framework.tasks.base.Language attribute) RMA (eval_framework.tasks.base.Language attribute) RMB (eval_framework.tasks.base.Language attribute) RMC (eval_framework.tasks.base.Language attribute) RMD (eval_framework.tasks.base.Language attribute) RME (eval_framework.tasks.base.Language attribute) RMF (eval_framework.tasks.base.Language attribute) RMG (eval_framework.tasks.base.Language attribute) RMH (eval_framework.tasks.base.Language attribute) RMI (eval_framework.tasks.base.Language attribute) RMK (eval_framework.tasks.base.Language attribute) RML (eval_framework.tasks.base.Language attribute) RMM (eval_framework.tasks.base.Language attribute) RMN (eval_framework.tasks.base.Language attribute) RMO (eval_framework.tasks.base.Language attribute) RMP (eval_framework.tasks.base.Language attribute) RMQ (eval_framework.tasks.base.Language attribute) RMR (eval_framework.tasks.base.Language attribute) RMS (eval_framework.tasks.base.Language attribute) RMT (eval_framework.tasks.base.Language attribute) RMU (eval_framework.tasks.base.Language attribute) RMV (eval_framework.tasks.base.Language attribute) RMW (eval_framework.tasks.base.Language attribute) RMX (eval_framework.tasks.base.Language attribute) RMY (eval_framework.tasks.base.Language attribute) RMZ (eval_framework.tasks.base.Language attribute) RNA (eval_framework.tasks.base.Language attribute) RNB (eval_framework.tasks.base.Language attribute) RND (eval_framework.tasks.base.Language attribute) RNG (eval_framework.tasks.base.Language attribute) RNL (eval_framework.tasks.base.Language attribute) RNN (eval_framework.tasks.base.Language attribute) RNP (eval_framework.tasks.base.Language attribute) RNR (eval_framework.tasks.base.Language attribute) RNW (eval_framework.tasks.base.Language attribute) ROB (eval_framework.tasks.base.Language attribute) ROC (eval_framework.tasks.base.Language attribute) ROD (eval_framework.tasks.base.Language attribute) ROE (eval_framework.tasks.base.Language attribute) ROF (eval_framework.tasks.base.Language attribute) ROG (eval_framework.tasks.base.Language attribute) ROH (eval_framework.tasks.base.Language attribute) ROL (eval_framework.tasks.base.Language attribute) ROM (eval_framework.tasks.base.Language attribute) RON (eval_framework.tasks.base.Language attribute) ROO (eval_framework.tasks.base.Language attribute) ROP (eval_framework.tasks.base.Language attribute) ROR (eval_framework.tasks.base.Language attribute) ROU (eval_framework.tasks.base.Language attribute) ROUGE_1 (class in eval_framework.metrics.completion.rouge_1) ROUGE_2 (class in eval_framework.metrics.completion.rouge_2) ROUGE_GEOMETRIC_MEAN (class in eval_framework.metrics.completion.rouge_geometric_mean) ROUGE_L (class in eval_framework.metrics.completion.rouge_l) ROW (eval_framework.tasks.base.Language attribute) RPN (eval_framework.tasks.base.Language attribute) RPT (eval_framework.tasks.base.Language attribute) RRI (eval_framework.tasks.base.Language attribute) RRM (eval_framework.tasks.base.Language attribute) RRO (eval_framework.tasks.base.Language attribute) RRT (eval_framework.tasks.base.Language attribute) RSB (eval_framework.tasks.base.Language attribute) RSI (eval_framework.tasks.base.Language attribute) RSK (eval_framework.tasks.base.Language attribute) RSL (eval_framework.tasks.base.Language attribute) RSM (eval_framework.tasks.base.Language attribute) RSN (eval_framework.tasks.base.Language attribute) RSW (eval_framework.tasks.base.Language attribute) RTC (eval_framework.tasks.base.Language attribute) RTH (eval_framework.tasks.base.Language attribute) RTM (eval_framework.tasks.base.Language attribute) RTS (eval_framework.tasks.base.Language attribute) RTW (eval_framework.tasks.base.Language attribute) RUB (eval_framework.tasks.base.Language attribute) RUC (eval_framework.tasks.base.Language attribute) RUE (eval_framework.tasks.base.Language attribute) RUF (eval_framework.tasks.base.Language attribute) RUG (eval_framework.tasks.base.Language attribute) RUH (eval_framework.tasks.base.Language attribute) RUI (eval_framework.tasks.base.Language attribute) RUK (eval_framework.tasks.base.Language attribute) RUN (eval_framework.tasks.base.Language attribute) run() (in module eval_framework.run) run_env (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) run_eval() (eval_framework.evaluation_generator.EvaluationGenerator method) run_python_code() (in module eval_framework.tasks.utils) run_with_kwargs() (in module eval_framework.run) RUO (eval_framework.tasks.base.Language attribute) RUP (eval_framework.tasks.base.Language attribute) RUQ (eval_framework.tasks.base.Language attribute) RUS (eval_framework.tasks.base.Language attribute) RUT (eval_framework.tasks.base.Language attribute) RUU (eval_framework.tasks.base.Language attribute) RUY (eval_framework.tasks.base.Language attribute) RUZ (eval_framework.tasks.base.Language attribute) RWA (eval_framework.tasks.base.Language attribute) RWK (eval_framework.tasks.base.Language attribute) RWL (eval_framework.tasks.base.Language attribute) RWM (eval_framework.tasks.base.Language attribute) RWO (eval_framework.tasks.base.Language attribute) RWR (eval_framework.tasks.base.Language attribute) RWS (eval_framework.tasks.base.Language attribute) RXD (eval_framework.tasks.base.Language attribute) RXW (eval_framework.tasks.base.Language attribute) RYN (eval_framework.tasks.base.Language attribute) RYS (eval_framework.tasks.base.Language attribute) RYU (eval_framework.tasks.base.Language attribute) RZH (eval_framework.tasks.base.Language attribute) S SAA (eval_framework.tasks.base.Language attribute) SAB (eval_framework.tasks.base.Language attribute) SAC (eval_framework.tasks.base.Language attribute) SAD (eval_framework.tasks.base.Language attribute) SAE (eval_framework.tasks.base.Language attribute) SAF (eval_framework.tasks.base.Language attribute) safe_json_loads() (in module eval_framework.llm.aleph_alpha) SAG (eval_framework.tasks.base.Language attribute) SAH (eval_framework.tasks.base.Language attribute) SAJ (eval_framework.tasks.base.Language attribute) SAK (eval_framework.tasks.base.Language attribute) SAM (eval_framework.tasks.base.Language attribute) Sample (class in eval_framework.tasks.base) SAMPLE_SPLIT (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeDebug attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeRun attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnDia attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnMC attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnQA attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_MathFind attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveKV2 attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveNumber attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrievePassKey1 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2025 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_FR attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winox.WINOX attribute) (eval_framework.tasks.benchmarks.wmt.WMT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_COMPLETION attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) SAN (eval_framework.tasks.base.Language attribute) SAO (eval_framework.tasks.base.Language attribute) SAP (eval_framework.tasks.base.Language attribute) SAQ (eval_framework.tasks.base.Language attribute) SAR (eval_framework.tasks.base.Language attribute) SAS (eval_framework.tasks.base.Language attribute) SAT (eval_framework.tasks.base.Language attribute) SAU (eval_framework.tasks.base.Language attribute) SAV (eval_framework.tasks.base.Language attribute) save() (eval_framework.base_config.BaseConfig method) save_aggregated_results() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_hf_dataset_commits() (in module eval_framework.tasks.task_names) save_intermediate_results (eval_framework.tasks.eval_config.EvalConfig attribute) save_logs (eval_framework.tasks.eval_config.EvalConfig attribute) save_metadata() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_metrics_result() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_metrics_results() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_response() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_responses() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) SAW (eval_framework.tasks.base.Language attribute) SAX (eval_framework.tasks.base.Language attribute) SAY (eval_framework.tasks.base.Language attribute) SAZ (eval_framework.tasks.base.Language attribute) SBA (eval_framework.tasks.base.Language attribute) SBB (eval_framework.tasks.base.Language attribute) SBC (eval_framework.tasks.base.Language attribute) SBD (eval_framework.tasks.base.Language attribute) SBE (eval_framework.tasks.base.Language attribute) SBF (eval_framework.tasks.base.Language attribute) SBG (eval_framework.tasks.base.Language attribute) SBH (eval_framework.tasks.base.Language attribute) SBI (eval_framework.tasks.base.Language attribute) SBJ (eval_framework.tasks.base.Language attribute) SBK (eval_framework.tasks.base.Language attribute) SBL (eval_framework.tasks.base.Language attribute) SBM (eval_framework.tasks.base.Language attribute) SBN (eval_framework.tasks.base.Language attribute) SBO (eval_framework.tasks.base.Language attribute) SBP (eval_framework.tasks.base.Language attribute) SBQ (eval_framework.tasks.base.Language attribute) SBR (eval_framework.tasks.base.Language attribute) SBS (eval_framework.tasks.base.Language attribute) SBT (eval_framework.tasks.base.Language attribute) SBU (eval_framework.tasks.base.Language attribute) SBV (eval_framework.tasks.base.Language attribute) SBW (eval_framework.tasks.base.Language attribute) SBX (eval_framework.tasks.base.Language attribute) SBY (eval_framework.tasks.base.Language attribute) SBZ (eval_framework.tasks.base.Language attribute) SCA (eval_framework.tasks.base.Language attribute) SCB (eval_framework.tasks.base.Language attribute) SCC (eval_framework.tasks.base.Language attribute) SCE (eval_framework.tasks.base.Language attribute) SCF (eval_framework.tasks.base.Language attribute) SCG (eval_framework.tasks.base.Language attribute) SCH (eval_framework.tasks.base.Language attribute) schema_error (eval_framework.metrics.llm.llm_judge_sql.SqlValidationResult attribute) schema_validation_error (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) SCI (eval_framework.tasks.base.Language attribute) SCIQ (class in eval_framework.tasks.benchmarks.sciq) SCIQ_IDK (class in eval_framework.tasks.benchmarks.sciq) SCIQEvalHarness (class in eval_framework.tasks.benchmarks.sciq) SCIQEvalHarness_IDK (class in eval_framework.tasks.benchmarks.sciq) SCK (eval_framework.tasks.base.Language attribute) SCL (eval_framework.tasks.base.Language attribute) SCN (eval_framework.tasks.base.Language attribute) SCO (eval_framework.tasks.base.Language attribute) SCP (eval_framework.tasks.base.Language attribute) SCQ (eval_framework.tasks.base.Language attribute) SCR (eval_framework.tasks.base.Language attribute) SCS (eval_framework.tasks.base.Language attribute) SCT (eval_framework.tasks.base.Language attribute) SCU (eval_framework.tasks.base.Language attribute) SCV (eval_framework.tasks.base.Language attribute) SCW (eval_framework.tasks.base.Language attribute) SCX (eval_framework.tasks.base.Language attribute) SCZ (eval_framework.tasks.base.Language attribute) SDA (eval_framework.tasks.base.Language attribute) SDB (eval_framework.tasks.base.Language attribute) SDC (eval_framework.tasks.base.Language attribute) SDD (eval_framework.tasks.base.Language attribute) SDE (eval_framework.tasks.base.Language attribute) SDF (eval_framework.tasks.base.Language attribute) SDG (eval_framework.tasks.base.Language attribute) SDH (eval_framework.tasks.base.Language attribute) SDI (eval_framework.tasks.base.Language attribute) SDJ (eval_framework.tasks.base.Language attribute) SDK (eval_framework.tasks.base.Language attribute) SDL (eval_framework.tasks.base.Language attribute) SDM (eval_framework.tasks.base.Language attribute) SDN (eval_framework.tasks.base.Language attribute) SDO (eval_framework.tasks.base.Language attribute) SDP (eval_framework.tasks.base.Language attribute) SDQ (eval_framework.tasks.base.Language attribute) SDR (eval_framework.tasks.base.Language attribute) SDS (eval_framework.tasks.base.Language attribute) SDT (eval_framework.tasks.base.Language attribute) SDU (eval_framework.tasks.base.Language attribute) SDX (eval_framework.tasks.base.Language attribute) SDZ (eval_framework.tasks.base.Language attribute) SEA (eval_framework.tasks.base.Language attribute) SEB (eval_framework.tasks.base.Language attribute) SEC (eval_framework.tasks.base.Language attribute) SED (eval_framework.tasks.base.Language attribute) SEE (eval_framework.tasks.base.Language attribute) seed (eval_framework.tasks.perturbation.PerturbationConfig attribute) SEF (eval_framework.tasks.base.Language attribute) SEG (eval_framework.tasks.base.Language attribute) SEH (eval_framework.tasks.base.Language attribute) SEI (eval_framework.tasks.base.Language attribute) SEJ (eval_framework.tasks.base.Language attribute) SEK (eval_framework.tasks.base.Language attribute) SEL (eval_framework.tasks.base.Language attribute) SEN (eval_framework.tasks.base.Language attribute) sentence_count (eval_framework.metrics.completion.text_counter.SentenceCounterMetricContext attribute) SentenceCounter (class in eval_framework.metrics.completion.text_counter) SentenceCounterMetricContext (class in eval_framework.metrics.completion.text_counter) SENTENCES (eval_framework.metrics.completion.length_control.LengthRequirementUnit attribute) SEO (eval_framework.tasks.base.Language attribute) SEP (eval_framework.tasks.base.Language attribute) separate_statements() (in module eval_framework.metrics.llm.llm_judge_sql) SEQ (eval_framework.tasks.base.Language attribute) SEQ_LENGTH (eval_framework.llm.huggingface.BaseHFLLM attribute) seq_length (eval_framework.llm.huggingface.BaseHFLLM property) SEQ_LENGTH (eval_framework.llm.vllm.BaseVLLMModel attribute) seq_length (eval_framework.llm.vllm.BaseVLLMModel property) SequencePositionsCompletion (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) SequencePositionsLoglikelihood (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) SER (eval_framework.tasks.base.Language attribute) SerializationError serialize_llm_class() (eval_framework.tasks.eval_config.EvalConfig method) serialize_llm_judge_class() (eval_framework.tasks.eval_config.EvalConfig method) serialize_output_dir() (eval_framework.tasks.eval_config.EvalConfig method) SES (eval_framework.tasks.base.Language attribute) SET (eval_framework.tasks.base.Language attribute) SEU (eval_framework.tasks.base.Language attribute) SEV (eval_framework.tasks.base.Language attribute) SEW (eval_framework.tasks.base.Language attribute) SEY (eval_framework.tasks.base.Language attribute) SEZ (eval_framework.tasks.base.Language attribute) SFB (eval_framework.tasks.base.Language attribute) SFE (eval_framework.tasks.base.Language attribute) SFM (eval_framework.tasks.base.Language attribute) SFS (eval_framework.tasks.base.Language attribute) SFW (eval_framework.tasks.base.Language attribute) SGA (eval_framework.tasks.base.Language attribute) SGB (eval_framework.tasks.base.Language attribute) SGC (eval_framework.tasks.base.Language attribute) SGD (eval_framework.tasks.base.Language attribute) SGE (eval_framework.tasks.base.Language attribute) SGG (eval_framework.tasks.base.Language attribute) SGH (eval_framework.tasks.base.Language attribute) SGI (eval_framework.tasks.base.Language attribute) SGJ (eval_framework.tasks.base.Language attribute) SGK (eval_framework.tasks.base.Language attribute) SGL (eval_framework.tasks.base.Language attribute) SGM (eval_framework.tasks.base.Language attribute) SGO (eval_framework.tasks.base.Language attribute) SGP (eval_framework.tasks.base.Language attribute) SGR (eval_framework.tasks.base.Language attribute) SGS (eval_framework.tasks.base.Language attribute) SGT (eval_framework.tasks.base.Language attribute) SGU (eval_framework.tasks.base.Language attribute) SGW (eval_framework.tasks.base.Language attribute) SGX (eval_framework.tasks.base.Language attribute) SGY (eval_framework.tasks.base.Language attribute) SGZ (eval_framework.tasks.base.Language attribute) SHA (eval_framework.tasks.base.Language attribute) SHB (eval_framework.tasks.base.Language attribute) SHC (eval_framework.tasks.base.Language attribute) SHD (eval_framework.tasks.base.Language attribute) SHE (eval_framework.tasks.base.Language attribute) SHG (eval_framework.tasks.base.Language attribute) SHH (eval_framework.tasks.base.Language attribute) SHI (eval_framework.tasks.base.Language attribute) SHJ (eval_framework.tasks.base.Language attribute) SHK (eval_framework.tasks.base.Language attribute) SHL (eval_framework.tasks.base.Language attribute) SHM (eval_framework.tasks.base.Language attribute) SHN (eval_framework.tasks.base.Language attribute) SHO (eval_framework.tasks.base.Language attribute) should_preempt() (eval_framework.context.determined.DeterminedContext method) (eval_framework.context.eval.EvalContext method) SHP (eval_framework.tasks.base.Language attribute) SHQ (eval_framework.tasks.base.Language attribute) SHR (eval_framework.tasks.base.Language attribute) SHS (eval_framework.tasks.base.Language attribute) SHT (eval_framework.tasks.base.Language attribute) SHU (eval_framework.tasks.base.Language attribute) SHV (eval_framework.tasks.base.Language attribute) SHW (eval_framework.tasks.base.Language attribute) SHX (eval_framework.tasks.base.Language attribute) SHY (eval_framework.tasks.base.Language attribute) SHZ (eval_framework.tasks.base.Language attribute) SIA (eval_framework.tasks.base.Language attribute) SIB (eval_framework.tasks.base.Language attribute) SIC (eval_framework.tasks.base.Language attribute) SID (eval_framework.tasks.base.Language attribute) SIE (eval_framework.tasks.base.Language attribute) SIF (eval_framework.tasks.base.Language attribute) SIG (eval_framework.tasks.base.Language attribute) SIH (eval_framework.tasks.base.Language attribute) SII (eval_framework.tasks.base.Language attribute) SIJ (eval_framework.tasks.base.Language attribute) SIK (eval_framework.tasks.base.Language attribute) SIL (eval_framework.tasks.base.Language attribute) SIM (eval_framework.tasks.base.Language attribute) SIN (eval_framework.tasks.base.Language attribute) SIP (eval_framework.tasks.base.Language attribute) SIQ (eval_framework.tasks.base.Language attribute) SIR (eval_framework.tasks.base.Language attribute) SIS (eval_framework.tasks.base.Language attribute) SIU (eval_framework.tasks.base.Language attribute) SIV (eval_framework.tasks.base.Language attribute) SIW (eval_framework.tasks.base.Language attribute) SIX (eval_framework.tasks.base.Language attribute) SIY (eval_framework.tasks.base.Language attribute) SIZ (eval_framework.tasks.base.Language attribute) SJA (eval_framework.tasks.base.Language attribute) SJB (eval_framework.tasks.base.Language attribute) SJC (eval_framework.tasks.base.Language attribute) SJD (eval_framework.tasks.base.Language attribute) SJE (eval_framework.tasks.base.Language attribute) SJG (eval_framework.tasks.base.Language attribute) SJK (eval_framework.tasks.base.Language attribute) SJL (eval_framework.tasks.base.Language attribute) SJM (eval_framework.tasks.base.Language attribute) SJN (eval_framework.tasks.base.Language attribute) SJO (eval_framework.tasks.base.Language attribute) SJP (eval_framework.tasks.base.Language attribute) SJR (eval_framework.tasks.base.Language attribute) SJS (eval_framework.tasks.base.Language attribute) SJT (eval_framework.tasks.base.Language attribute) SJU (eval_framework.tasks.base.Language attribute) SJW (eval_framework.tasks.base.Language attribute) SKA (eval_framework.tasks.base.Language attribute) SKB (eval_framework.tasks.base.Language attribute) SKC (eval_framework.tasks.base.Language attribute) SKD (eval_framework.tasks.base.Language attribute) SKE (eval_framework.tasks.base.Language attribute) SKF (eval_framework.tasks.base.Language attribute) SKG (eval_framework.tasks.base.Language attribute) SKH (eval_framework.tasks.base.Language attribute) SKI (eval_framework.tasks.base.Language attribute) SKJ (eval_framework.tasks.base.Language attribute) SKK (eval_framework.tasks.base.Language attribute) SKL (eval_framework.tasks.base.Language attribute) SKM (eval_framework.tasks.base.Language attribute) SKN (eval_framework.tasks.base.Language attribute) SKO (eval_framework.tasks.base.Language attribute) SKP (eval_framework.tasks.base.Language attribute) SKQ (eval_framework.tasks.base.Language attribute) SKR (eval_framework.tasks.base.Language attribute) SKS (eval_framework.tasks.base.Language attribute) SKT (eval_framework.tasks.base.Language attribute) SKU (eval_framework.tasks.base.Language attribute) SKV (eval_framework.tasks.base.Language attribute) SKW (eval_framework.tasks.base.Language attribute) SKX (eval_framework.tasks.base.Language attribute) SKY (eval_framework.tasks.base.Language attribute) SKZ (eval_framework.tasks.base.Language attribute) SLB (eval_framework.tasks.base.Language attribute) SLC (eval_framework.tasks.base.Language attribute) SLD (eval_framework.tasks.base.Language attribute) SLE (eval_framework.tasks.base.Language attribute) SLF (eval_framework.tasks.base.Language attribute) SLG (eval_framework.tasks.base.Language attribute) SLH (eval_framework.tasks.base.Language attribute) SLI (eval_framework.tasks.base.Language attribute) SLJ (eval_framework.tasks.base.Language attribute) SLK (eval_framework.tasks.base.Language attribute) SLL (eval_framework.tasks.base.Language attribute) SLM (eval_framework.tasks.base.Language attribute) SLN (eval_framework.tasks.base.Language attribute) SLP (eval_framework.tasks.base.Language attribute) SLQ (eval_framework.tasks.base.Language attribute) SLR (eval_framework.tasks.base.Language attribute) SLS (eval_framework.tasks.base.Language attribute) SLT (eval_framework.tasks.base.Language attribute) SLU (eval_framework.tasks.base.Language attribute) SLV (eval_framework.tasks.base.Language attribute) SLW (eval_framework.tasks.base.Language attribute) SLX (eval_framework.tasks.base.Language attribute) SLY (eval_framework.tasks.base.Language attribute) SLZ (eval_framework.tasks.base.Language attribute) SMA (eval_framework.tasks.base.Language attribute) SMB (eval_framework.tasks.base.Language attribute) SMC (eval_framework.tasks.base.Language attribute) SMD (eval_framework.tasks.base.Language attribute) SME (eval_framework.tasks.base.Language attribute) SMF (eval_framework.tasks.base.Language attribute) SMG (eval_framework.tasks.base.Language attribute) SMH (eval_framework.tasks.base.Language attribute) SMJ (eval_framework.tasks.base.Language attribute) SMK (eval_framework.tasks.base.Language attribute) SML (eval_framework.tasks.base.Language attribute) SMM (eval_framework.tasks.base.Language attribute) SMN (eval_framework.tasks.base.Language attribute) SMO (eval_framework.tasks.base.Language attribute) SmolLM135M (class in eval_framework.llm.huggingface) Smollm135MInstruct (class in eval_framework.llm.huggingface) SMP (eval_framework.tasks.base.Language attribute) SMQ (eval_framework.tasks.base.Language attribute) SMR (eval_framework.tasks.base.Language attribute) SMS (eval_framework.tasks.base.Language attribute) SMT (eval_framework.tasks.base.Language attribute) SMU (eval_framework.tasks.base.Language attribute) SMV (eval_framework.tasks.base.Language attribute) SMW (eval_framework.tasks.base.Language attribute) SMX (eval_framework.tasks.base.Language attribute) SMY (eval_framework.tasks.base.Language attribute) SMZ (eval_framework.tasks.base.Language attribute) SNA (eval_framework.tasks.base.Language attribute) SNB (eval_framework.tasks.base.Language attribute) SNC (eval_framework.tasks.base.Language attribute) SND (eval_framework.tasks.base.Language attribute) SNE (eval_framework.tasks.base.Language attribute) SNF (eval_framework.tasks.base.Language attribute) SNG (eval_framework.tasks.base.Language attribute) SNH (eval_framework.tasks.base.Language attribute) SNI (eval_framework.tasks.base.Language attribute) snippet_merge_fn (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext attribute) SNJ (eval_framework.tasks.base.Language attribute) SNK (eval_framework.tasks.base.Language attribute) SNL (eval_framework.tasks.base.Language attribute) SNM (eval_framework.tasks.base.Language attribute) SNN (eval_framework.tasks.base.Language attribute) SNO (eval_framework.tasks.base.Language attribute) SNP (eval_framework.tasks.base.Language attribute) SNQ (eval_framework.tasks.base.Language attribute) SNR (eval_framework.tasks.base.Language attribute) SNS (eval_framework.tasks.base.Language attribute) SNU (eval_framework.tasks.base.Language attribute) SNV (eval_framework.tasks.base.Language attribute) SNW (eval_framework.tasks.base.Language attribute) SNX (eval_framework.tasks.base.Language attribute) SNY (eval_framework.tasks.base.Language attribute) SNZ (eval_framework.tasks.base.Language attribute) SOA (eval_framework.tasks.base.Language attribute) SOB (eval_framework.tasks.base.Language attribute) SOC (eval_framework.tasks.base.Language attribute) SOD (eval_framework.tasks.base.Language attribute) SOE (eval_framework.tasks.base.Language attribute) SOG (eval_framework.tasks.base.Language attribute) SOH (eval_framework.tasks.base.Language attribute) SOI (eval_framework.tasks.base.Language attribute) SOJ (eval_framework.tasks.base.Language attribute) SOK (eval_framework.tasks.base.Language attribute) SOL (eval_framework.tasks.base.Language attribute) SOM (eval_framework.tasks.base.Language attribute) SOO (eval_framework.tasks.base.Language attribute) SOP (eval_framework.tasks.base.Language attribute) SOQ (eval_framework.tasks.base.Language attribute) SOR (eval_framework.tasks.base.Language attribute) SOS (eval_framework.tasks.base.Language attribute) SOT (eval_framework.tasks.base.Language attribute) SOU (eval_framework.tasks.base.Language attribute) SOV (eval_framework.tasks.base.Language attribute) SOW (eval_framework.tasks.base.Language attribute) SOX (eval_framework.tasks.base.Language attribute) SOY (eval_framework.tasks.base.Language attribute) SOZ (eval_framework.tasks.base.Language attribute) SPA (eval_framework.tasks.base.Language attribute) SPB (eval_framework.tasks.base.Language attribute) SPC (eval_framework.tasks.base.Language attribute) SPD (eval_framework.tasks.base.Language attribute) SPE (eval_framework.tasks.base.Language attribute) SPG (eval_framework.tasks.base.Language attribute) SPHYR (class in eval_framework.tasks.benchmarks.sphyr) SPI (eval_framework.tasks.base.Language attribute) SPK (eval_framework.tasks.base.Language attribute) SPL (eval_framework.tasks.base.Language attribute) SPM (eval_framework.tasks.base.Language attribute) SPN (eval_framework.tasks.base.Language attribute) SPO (eval_framework.tasks.base.Language attribute) SPP (eval_framework.tasks.base.Language attribute) SPQ (eval_framework.tasks.base.Language attribute) SPR (eval_framework.tasks.base.Language attribute) SPS (eval_framework.tasks.base.Language attribute) SPT (eval_framework.tasks.base.Language attribute) SPU (eval_framework.tasks.base.Language attribute) SPV (eval_framework.tasks.base.Language attribute) SPX (eval_framework.tasks.base.Language attribute) SPY (eval_framework.tasks.base.Language attribute) SQA (eval_framework.tasks.base.Language attribute) SQH (eval_framework.tasks.base.Language attribute) SQI (eval_framework.tasks.base.Language attribute) SQK (eval_framework.tasks.base.Language attribute) SqlDialects (class in eval_framework.metrics.llm.llm_judge_sql) sqlite (eval_framework.metrics.llm.llm_judge_sql.SqlDialects attribute) SqlOutputComparison (class in eval_framework.metrics.llm.llm_judge_sql) SqlValidationResult (class in eval_framework.metrics.llm.llm_judge_sql) SQM (eval_framework.tasks.base.Language attribute) SQN (eval_framework.tasks.base.Language attribute) SQO (eval_framework.tasks.base.Language attribute) SQQ (eval_framework.tasks.base.Language attribute) SQR (eval_framework.tasks.base.Language attribute) SQS (eval_framework.tasks.base.Language attribute) SQT (eval_framework.tasks.base.Language attribute) SQU (eval_framework.tasks.base.Language attribute) SQUAD (class in eval_framework.tasks.benchmarks.squad) SQUAD2 (class in eval_framework.tasks.benchmarks.squad) SQX (eval_framework.tasks.base.Language attribute) SRA (eval_framework.tasks.base.Language attribute) SRB (eval_framework.tasks.base.Language attribute) SRC (eval_framework.tasks.base.Language attribute) SRD (eval_framework.tasks.base.Language attribute) SRE (eval_framework.tasks.base.Language attribute) SRF (eval_framework.tasks.base.Language attribute) SRG (eval_framework.tasks.base.Language attribute) SRH (eval_framework.tasks.base.Language attribute) SRI (eval_framework.tasks.base.Language attribute) SRJ (eval_framework.tasks.base.Language attribute) SRK (eval_framework.tasks.base.Language attribute) SRL (eval_framework.tasks.base.Language attribute) SRM (eval_framework.tasks.base.Language attribute) SRN (eval_framework.tasks.base.Language attribute) SRO (eval_framework.tasks.base.Language attribute) SRP (eval_framework.tasks.base.Language attribute) SRQ (eval_framework.tasks.base.Language attribute) SRR (eval_framework.tasks.base.Language attribute) SRS (eval_framework.tasks.base.Language attribute) SRT (eval_framework.tasks.base.Language attribute) SRU (eval_framework.tasks.base.Language attribute) SRV (eval_framework.tasks.base.Language attribute) SRW (eval_framework.tasks.base.Language attribute) SRX (eval_framework.tasks.base.Language attribute) SRY (eval_framework.tasks.base.Language attribute) SRZ (eval_framework.tasks.base.Language attribute) SSB (eval_framework.tasks.base.Language attribute) SSC (eval_framework.tasks.base.Language attribute) SSD (eval_framework.tasks.base.Language attribute) SSE (eval_framework.tasks.base.Language attribute) SSF (eval_framework.tasks.base.Language attribute) SSG (eval_framework.tasks.base.Language attribute) SSH (eval_framework.tasks.base.Language attribute) SSI (eval_framework.tasks.base.Language attribute) SSJ (eval_framework.tasks.base.Language attribute) SSK (eval_framework.tasks.base.Language attribute) SSL (eval_framework.tasks.base.Language attribute) SSM (eval_framework.tasks.base.Language attribute) SSN (eval_framework.tasks.base.Language attribute) SSO (eval_framework.tasks.base.Language attribute) SSP (eval_framework.tasks.base.Language attribute) SSQ (eval_framework.tasks.base.Language attribute) SSR (eval_framework.tasks.base.Language attribute) SSS (eval_framework.tasks.base.Language attribute) SST (eval_framework.tasks.base.Language attribute) SSU (eval_framework.tasks.base.Language attribute) SSV (eval_framework.tasks.base.Language attribute) SSW (eval_framework.tasks.base.Language attribute) SSX (eval_framework.tasks.base.Language attribute) SSY (eval_framework.tasks.base.Language attribute) SSZ (eval_framework.tasks.base.Language attribute) STA (eval_framework.tasks.base.Language attribute) standard_sql (eval_framework.metrics.llm.llm_judge_sql.SqlDialects attribute) STB (eval_framework.tasks.base.Language attribute) STC (eval_framework.tasks.base.Language attribute) STD (eval_framework.tasks.base.Language attribute) STE (eval_framework.tasks.base.Language attribute) STF (eval_framework.tasks.base.Language attribute) STG (eval_framework.tasks.base.Language attribute) STH (eval_framework.tasks.base.Language attribute) STI (eval_framework.tasks.base.Language attribute) STJ (eval_framework.tasks.base.Language attribute) STK (eval_framework.tasks.base.Language attribute) STL (eval_framework.tasks.base.Language attribute) STM (eval_framework.tasks.base.Language attribute) STN (eval_framework.tasks.base.Language attribute) STO (eval_framework.tasks.base.Language attribute) stop_sequences (eval_framework.tasks.benchmarks.gpqa.GPQA_COT attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_CodeRun attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnDia attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_EnQA attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_MathFind attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveKV2 attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrieveNumber attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench_RetrievePassKey1 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU_COT attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.wmt.WMT_INSTRUCT attribute) StopSequenceCriteria (class in eval_framework.llm.huggingface) STP (eval_framework.tasks.base.Language attribute) STQ (eval_framework.tasks.base.Language attribute) STR (eval_framework.tasks.base.Language attribute) StructEval (class in eval_framework.tasks.benchmarks.struct_eval) StructMetric (class in eval_framework.metrics.completion.struct_eval_metrics) StructMetricContext (class in eval_framework.metrics.completion.struct_eval_metrics) STS (eval_framework.tasks.base.Language attribute) STT (eval_framework.tasks.base.Language attribute) STU (eval_framework.tasks.base.Language attribute) STV (eval_framework.tasks.base.Language attribute) STW (eval_framework.tasks.base.Language attribute) STY (eval_framework.tasks.base.Language attribute) SUA (eval_framework.tasks.base.Language attribute) SUB (eval_framework.tasks.base.Language attribute) subject (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.base.Sample attribute) subject_to_language() (in module eval_framework.tasks.benchmarks.include) SUBJECTS (eval_framework.tasks.base.BaseTask attribute) (eval_framework.tasks.benchmarks.aidanbench.AidanBenchOriginal attribute) (eval_framework.tasks.benchmarks.arc.ARC attribute) (eval_framework.tasks.benchmarks.arc_de.ARC_DE attribute) (eval_framework.tasks.benchmarks.arc_fi.ARC_FI attribute) (eval_framework.tasks.benchmarks.belebele.BELEBELE attribute) (eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench attribute) (eval_framework.tasks.benchmarks.casehold.CASEHOLD attribute) (eval_framework.tasks.benchmarks.chembench.ChemBench attribute) (eval_framework.tasks.benchmarks.copa.COPA attribute) (eval_framework.tasks.benchmarks.duc.DUC attribute) (eval_framework.tasks.benchmarks.duc.DUC_ABSTRACTIVE attribute) (eval_framework.tasks.benchmarks.duc.DUC_EXTRACTIVE attribute) (eval_framework.tasks.benchmarks.flores200.Flores200 attribute) (eval_framework.tasks.benchmarks.flores_plus.FloresPlus attribute) (eval_framework.tasks.benchmarks.gpqa.GPQA attribute) (eval_framework.tasks.benchmarks.gsm8k.GSM8KEvalHarness attribute) (eval_framework.tasks.benchmarks.hellaswag.HELLASWAG attribute) (eval_framework.tasks.benchmarks.hellaswag_de.HELLASWAG_DE attribute) (eval_framework.tasks.benchmarks.humaneval.HumanEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEval attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalDe attribute) (eval_framework.tasks.benchmarks.ifeval.IFEvalFiSv attribute) (eval_framework.tasks.benchmarks.include.INCLUDE attribute) (eval_framework.tasks.benchmarks.infinitebench.InfiniteBench attribute) (eval_framework.tasks.benchmarks.math_reasoning.AIME2024 attribute) (eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATH500 attribute) (eval_framework.tasks.benchmarks.math_reasoning.MATHReasoning attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP_PROMPT_WITHOUT_TESTS_SANITIZED attribute) (eval_framework.tasks.benchmarks.mbpp.MBPP_SANITIZED attribute) (eval_framework.tasks.benchmarks.mmlu.MMLU attribute) (eval_framework.tasks.benchmarks.mmlu_de.MMLU_DE attribute) (eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU attribute) (eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT attribute) (eval_framework.tasks.benchmarks.openbookqa.OPENBOOKQA attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.ARC_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.GSM8K_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.HELLASWAG_EU20_FR attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_DE attribute) (eval_framework.tasks.benchmarks.opengptx_eu20.MMLU_EU20_FR attribute) (eval_framework.tasks.benchmarks.pawsx.PAWSX attribute) (eval_framework.tasks.benchmarks.piqa.PIQA attribute) (eval_framework.tasks.benchmarks.quality.QUALITY attribute) (eval_framework.tasks.benchmarks.sciq.SCIQ attribute) (eval_framework.tasks.benchmarks.sciq.SCIQEvalHarness attribute) (eval_framework.tasks.benchmarks.sphyr.SPHYR attribute) (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) (eval_framework.tasks.benchmarks.struct_eval.RenderableStructEval attribute) (eval_framework.tasks.benchmarks.struct_eval.StructEval attribute) (eval_framework.tasks.benchmarks.tablebench.TableBench attribute) (eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA attribute) (eval_framework.tasks.benchmarks.truthfulqa.TRUTHFULQA attribute) (eval_framework.tasks.benchmarks.winogender.WINOGENDER attribute) (eval_framework.tasks.benchmarks.winogrande.WINOGRANDE attribute) (eval_framework.tasks.benchmarks.winox.WINOX_DE attribute) (eval_framework.tasks.benchmarks.winox.WINOX_FR attribute) (eval_framework.tasks.benchmarks.wmt.WMT14 attribute) (eval_framework.tasks.benchmarks.wmt.WMT14_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT16 attribute) (eval_framework.tasks.benchmarks.wmt.WMT16_INSTRUCT attribute) (eval_framework.tasks.benchmarks.wmt.WMT20 attribute) (eval_framework.tasks.benchmarks.wmt.WMT20_INSTRUCT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_GOV_REPORT attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_MUSIQUE attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_NARRATIVEQA attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QASPER attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QMSUM attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_QUALITY attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SPACE_DIGEST attribute) (eval_framework.tasks.benchmarks.zero_scrolls.ZERO_SCROLLS_SQUALITY attribute) SUBSTITUTIONS (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) SUC (eval_framework.tasks.base.Language attribute) success (eval_framework.metrics.llm.llm_judge_sql.SqlValidationResult attribute) (eval_framework.tasks.utils.ExecutionResult attribute), [1] SUE (eval_framework.tasks.base.Language attribute) SUF (eval_framework.tasks.base.Language attribute) SUG (eval_framework.tasks.base.Language attribute) SUH (eval_framework.tasks.base.Language attribute) SUI (eval_framework.tasks.base.Language attribute) SUJ (eval_framework.tasks.base.Language attribute) SUK (eval_framework.tasks.base.Language attribute) SUL (eval_framework.tasks.base.Language attribute) SUM (eval_framework.tasks.base.Language attribute) SUN (eval_framework.tasks.base.Language attribute) SUO (eval_framework.tasks.base.Language attribute) SUQ (eval_framework.tasks.base.Language attribute) SUR (eval_framework.tasks.base.Language attribute) SUS (eval_framework.tasks.base.Language attribute) SUT (eval_framework.tasks.base.Language attribute) SUU (eval_framework.tasks.base.Language attribute) SUV (eval_framework.tasks.base.Language attribute) SUW (eval_framework.tasks.base.Language attribute) SUX (eval_framework.tasks.base.Language attribute) SUY (eval_framework.tasks.base.Language attribute) SUZ (eval_framework.tasks.base.Language attribute) SVA (eval_framework.tasks.base.Language attribute) SVB (eval_framework.tasks.base.Language attribute) SVC (eval_framework.tasks.base.Language attribute) SVE (eval_framework.tasks.base.Language attribute) SVK (eval_framework.tasks.base.Language attribute) SVM (eval_framework.tasks.base.Language attribute) SVR (eval_framework.tasks.base.Language attribute) SVS (eval_framework.tasks.base.Language attribute) SVX (eval_framework.tasks.base.Language attribute) SWA (eval_framework.tasks.base.Language attribute) SWB (eval_framework.tasks.base.Language attribute) SWC (eval_framework.tasks.base.Language attribute) SWE (eval_framework.tasks.base.Language attribute) SWF (eval_framework.tasks.base.Language attribute) SWG (eval_framework.tasks.base.Language attribute) SWH (eval_framework.tasks.base.Language attribute) SWI (eval_framework.tasks.base.Language attribute) SWJ (eval_framework.tasks.base.Language attribute) SWK (eval_framework.tasks.base.Language attribute) SWL (eval_framework.tasks.base.Language attribute) SWM (eval_framework.tasks.base.Language attribute) SWN (eval_framework.tasks.base.Language attribute) SWO (eval_framework.tasks.base.Language attribute) SWP (eval_framework.tasks.base.Language attribute) SWQ (eval_framework.tasks.base.Language attribute) SWR (eval_framework.tasks.base.Language attribute) SWS (eval_framework.tasks.base.Language attribute) SWT (eval_framework.tasks.base.Language attribute) SWU (eval_framework.tasks.base.Language attribute) SWV (eval_framework.tasks.base.Language attribute) SWW (eval_framework.tasks.base.Language attribute) SWX (eval_framework.tasks.base.Language attribute) SWY (eval_framework.tasks.base.Language attribute) SXB (eval_framework.tasks.base.Language attribute) SXC (eval_framework.tasks.base.Language attribute) SXE (eval_framework.tasks.base.Language attribute) SXG (eval_framework.tasks.base.Language attribute) SXK (eval_framework.tasks.base.Language attribute) SXL (eval_framework.tasks.base.Language attribute) SXM (eval_framework.tasks.base.Language attribute) SXN (eval_framework.tasks.base.Language attribute) SXO (eval_framework.tasks.base.Language attribute) SXR (eval_framework.tasks.base.Language attribute) SXS (eval_framework.tasks.base.Language attribute) SXU (eval_framework.tasks.base.Language attribute) SXW (eval_framework.tasks.base.Language attribute) SYA (eval_framework.tasks.base.Language attribute) SYB (eval_framework.tasks.base.Language attribute) SYC (eval_framework.tasks.base.Language attribute) SYI (eval_framework.tasks.base.Language attribute) SYK (eval_framework.tasks.base.Language attribute) SYL (eval_framework.tasks.base.Language attribute) SYM (eval_framework.tasks.base.Language attribute) SYN (eval_framework.tasks.base.Language attribute) SYO (eval_framework.tasks.base.Language attribute) SYR (eval_framework.tasks.base.Language attribute) SYS (eval_framework.tasks.base.Language attribute) SYW (eval_framework.tasks.base.Language attribute) SYX (eval_framework.tasks.base.Language attribute) SYY (eval_framework.tasks.base.Language attribute) SZA (eval_framework.tasks.base.Language attribute) SZB (eval_framework.tasks.base.Language attribute) SZC (eval_framework.tasks.base.Language attribute) SZD (eval_framework.tasks.base.Language attribute) SZE (eval_framework.tasks.base.Language attribute) SZG (eval_framework.tasks.base.Language attribute) SZK (eval_framework.tasks.base.Language attribute) SZL (eval_framework.tasks.base.Language attribute) SZN (eval_framework.tasks.base.Language attribute) SZP (eval_framework.tasks.base.Language attribute) SZS (eval_framework.tasks.base.Language attribute) SZV (eval_framework.tasks.base.Language attribute) SZW (eval_framework.tasks.base.Language attribute) SZY (eval_framework.tasks.base.Language attribute) T TAA (eval_framework.tasks.base.Language attribute) TAB (eval_framework.tasks.base.Language attribute) TableBench (class in eval_framework.tasks.benchmarks.tablebench) TAC (eval_framework.tasks.base.Language attribute) TAD (eval_framework.tasks.base.Language attribute) TAE (eval_framework.tasks.base.Language attribute) TAF (eval_framework.tasks.base.Language attribute) TAG (eval_framework.tasks.base.Language attribute) TAH (eval_framework.tasks.base.Language attribute) TAJ (eval_framework.tasks.base.Language attribute) TAK (eval_framework.tasks.base.Language attribute) TAL (eval_framework.tasks.base.Language attribute) TAM (eval_framework.tasks.base.Language attribute) TAN (eval_framework.tasks.base.Language attribute) TAO (eval_framework.tasks.base.Language attribute) TAP (eval_framework.tasks.base.Language attribute) TAQ (eval_framework.tasks.base.Language attribute) TAR (eval_framework.tasks.base.Language attribute) TARGET (eval_framework.metrics.completion.length_control.LengthRequirementType attribute) TAS (eval_framework.tasks.base.Language attribute) task_args (eval_framework.context.determined.Hyperparameters attribute) task_class (eval_framework.tasks.eval_config.EvalConfig property) task_name (eval_framework.context.determined.TaskArgs attribute) (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) task_subjects (eval_framework.context.determined.TaskArgs attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) TaskArgs (class in eval_framework.context.determined) TaskNameEnum (class in eval_framework.tasks.task_names) TAT (eval_framework.tasks.base.Language attribute) TAU (eval_framework.tasks.base.Language attribute) TAV (eval_framework.tasks.base.Language attribute) TAW (eval_framework.tasks.base.Language attribute) TAX (eval_framework.tasks.base.Language attribute) TAY (eval_framework.tasks.base.Language attribute) TAZ (eval_framework.tasks.base.Language attribute) TBA (eval_framework.tasks.base.Language attribute) TBB (eval_framework.tasks.base.Language attribute) TBC (eval_framework.tasks.base.Language attribute) TBD (eval_framework.tasks.base.Language attribute) TBE (eval_framework.tasks.base.Language attribute) TBF (eval_framework.tasks.base.Language attribute) TBG (eval_framework.tasks.base.Language attribute) TBH (eval_framework.tasks.base.Language attribute) TBI (eval_framework.tasks.base.Language attribute) TBJ (eval_framework.tasks.base.Language attribute) TBK (eval_framework.tasks.base.Language attribute) TBL (eval_framework.tasks.base.Language attribute) TBM (eval_framework.tasks.base.Language attribute) TBN (eval_framework.tasks.base.Language attribute) TBO (eval_framework.tasks.base.Language attribute) TBP (eval_framework.tasks.base.Language attribute) TBR (eval_framework.tasks.base.Language attribute) TBS (eval_framework.tasks.base.Language attribute) TBT (eval_framework.tasks.base.Language attribute) TBU (eval_framework.tasks.base.Language attribute) TBV (eval_framework.tasks.base.Language attribute) TBW (eval_framework.tasks.base.Language attribute) TBX (eval_framework.tasks.base.Language attribute) TBY (eval_framework.tasks.base.Language attribute) TBZ (eval_framework.tasks.base.Language attribute) TCA (eval_framework.tasks.base.Language attribute) TCB (eval_framework.tasks.base.Language attribute) TCC (eval_framework.tasks.base.Language attribute) TCD (eval_framework.tasks.base.Language attribute) TCE (eval_framework.tasks.base.Language attribute) TCF (eval_framework.tasks.base.Language attribute) TCG (eval_framework.tasks.base.Language attribute) TCH (eval_framework.tasks.base.Language attribute) TCI (eval_framework.tasks.base.Language attribute) TCK (eval_framework.tasks.base.Language attribute) TCL (eval_framework.tasks.base.Language attribute) TCM (eval_framework.tasks.base.Language attribute) TCN (eval_framework.tasks.base.Language attribute) TCO (eval_framework.tasks.base.Language attribute) TCP (eval_framework.tasks.base.Language attribute) TCQ (eval_framework.tasks.base.Language attribute) TCS (eval_framework.tasks.base.Language attribute) TCT (eval_framework.tasks.base.Language attribute) TCU (eval_framework.tasks.base.Language attribute) TCW (eval_framework.tasks.base.Language attribute) TCX (eval_framework.tasks.base.Language attribute) TCY (eval_framework.tasks.base.Language attribute) TCZ (eval_framework.tasks.base.Language attribute) TDA (eval_framework.tasks.base.Language attribute) TDB (eval_framework.tasks.base.Language attribute) TDC (eval_framework.tasks.base.Language attribute) TDD (eval_framework.tasks.base.Language attribute) TDE (eval_framework.tasks.base.Language attribute) TDF (eval_framework.tasks.base.Language attribute) TDG (eval_framework.tasks.base.Language attribute) TDH (eval_framework.tasks.base.Language attribute) TDI (eval_framework.tasks.base.Language attribute) TDJ (eval_framework.tasks.base.Language attribute) TDK (eval_framework.tasks.base.Language attribute) TDL (eval_framework.tasks.base.Language attribute) TDM (eval_framework.tasks.base.Language attribute) TDN (eval_framework.tasks.base.Language attribute) TDO (eval_framework.tasks.base.Language attribute) TDQ (eval_framework.tasks.base.Language attribute) TDR (eval_framework.tasks.base.Language attribute) TDS (eval_framework.tasks.base.Language attribute) TDT (eval_framework.tasks.base.Language attribute) TDU (eval_framework.tasks.base.Language attribute) TDV (eval_framework.tasks.base.Language attribute) TDX (eval_framework.tasks.base.Language attribute) TDY (eval_framework.tasks.base.Language attribute) TEA (eval_framework.tasks.base.Language attribute) TEB (eval_framework.tasks.base.Language attribute) TEC (eval_framework.tasks.base.Language attribute) TED (eval_framework.tasks.base.Language attribute) TEE (eval_framework.tasks.base.Language attribute) TEF (eval_framework.tasks.base.Language attribute) TEG (eval_framework.tasks.base.Language attribute) TEH (eval_framework.tasks.base.Language attribute) TEI (eval_framework.tasks.base.Language attribute) TEK (eval_framework.tasks.base.Language attribute) TEL (eval_framework.tasks.base.Language attribute) TEM (eval_framework.tasks.base.Language attribute) TEN (eval_framework.tasks.base.Language attribute) TEO (eval_framework.tasks.base.Language attribute) TEP (eval_framework.tasks.base.Language attribute) TEQ (eval_framework.tasks.base.Language attribute) TER (class in eval_framework.metrics.completion.ter) (eval_framework.tasks.base.Language attribute) TernaryScore (class in eval_framework.metrics.loglikelihood.ternary) TES (eval_framework.tasks.base.Language attribute) test (eval_framework.tasks.benchmarks.humaneval.HumanEvalMetricContext attribute) test_code (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) tests_code (eval_framework.tasks.benchmarks.mbpp.MBPPMetricContext attribute) TET (eval_framework.tasks.base.Language attribute) TEU (eval_framework.tasks.base.Language attribute) TEV (eval_framework.tasks.base.Language attribute) TEW (eval_framework.tasks.base.Language attribute) TEX (eval_framework.tasks.base.Language attribute) text (eval_framework.llm.vllm.TokenizedContainer attribute) TEY (eval_framework.tasks.base.Language attribute) TEZ (eval_framework.tasks.base.Language attribute) TFI (eval_framework.tasks.base.Language attribute) TFN (eval_framework.tasks.base.Language attribute) TFO (eval_framework.tasks.base.Language attribute) TFR (eval_framework.tasks.base.Language attribute) TFT (eval_framework.tasks.base.Language attribute) TGA (eval_framework.tasks.base.Language attribute) TGB (eval_framework.tasks.base.Language attribute) TGC (eval_framework.tasks.base.Language attribute) TGD (eval_framework.tasks.base.Language attribute) TGE (eval_framework.tasks.base.Language attribute) TGF (eval_framework.tasks.base.Language attribute) TGG (eval_framework.tasks.base.Language attribute) TGH (eval_framework.tasks.base.Language attribute) TGI (eval_framework.tasks.base.Language attribute) TGJ (eval_framework.tasks.base.Language attribute) TGK (eval_framework.tasks.base.Language attribute) TGL (eval_framework.tasks.base.Language attribute) TGN (eval_framework.tasks.base.Language attribute) TGO (eval_framework.tasks.base.Language attribute) TGP (eval_framework.tasks.base.Language attribute) TGQ (eval_framework.tasks.base.Language attribute) TGR (eval_framework.tasks.base.Language attribute) TGS (eval_framework.tasks.base.Language attribute) TGT (eval_framework.tasks.base.Language attribute) TGU (eval_framework.tasks.base.Language attribute) TGV (eval_framework.tasks.base.Language attribute) TGW (eval_framework.tasks.base.Language attribute) TGX (eval_framework.tasks.base.Language attribute) TGY (eval_framework.tasks.base.Language attribute) TGZ (eval_framework.tasks.base.Language attribute) THA (eval_framework.tasks.base.Language attribute) THC (eval_framework.tasks.base.Language attribute) THD (eval_framework.tasks.base.Language attribute) THE (eval_framework.tasks.base.Language attribute) THF (eval_framework.tasks.base.Language attribute) THH (eval_framework.tasks.base.Language attribute) THI (eval_framework.tasks.base.Language attribute) THK (eval_framework.tasks.base.Language attribute) THL (eval_framework.tasks.base.Language attribute) THM (eval_framework.tasks.base.Language attribute) THN (eval_framework.tasks.base.Language attribute) THP (eval_framework.tasks.base.Language attribute) THQ (eval_framework.tasks.base.Language attribute) THR (eval_framework.tasks.base.Language attribute) THS (eval_framework.tasks.base.Language attribute) THT (eval_framework.tasks.base.Language attribute) THU (eval_framework.tasks.base.Language attribute) THV (eval_framework.tasks.base.Language attribute) THW (eval_framework.tasks.base.Language attribute) THX (eval_framework.tasks.base.Language attribute) THY (eval_framework.tasks.base.Language attribute) THZ (eval_framework.tasks.base.Language attribute) TIA (eval_framework.tasks.base.Language attribute) TIC (eval_framework.tasks.base.Language attribute) TID (eval_framework.tasks.base.Language attribute) TIE (eval_framework.tasks.base.Language attribute) TIF (eval_framework.tasks.base.Language attribute) TIG (eval_framework.tasks.base.Language attribute) TIH (eval_framework.tasks.base.Language attribute) TII (eval_framework.tasks.base.Language attribute) TIJ (eval_framework.tasks.base.Language attribute) TIK (eval_framework.tasks.base.Language attribute) TIL (eval_framework.tasks.base.Language attribute) TIM (eval_framework.tasks.base.Language attribute) timeout_handler() (in module eval_framework.metrics.completion.math_reasoning_completion) TIN (eval_framework.tasks.base.Language attribute) TIO (eval_framework.tasks.base.Language attribute) TIP (eval_framework.tasks.base.Language attribute) TIQ (eval_framework.tasks.base.Language attribute) TIR (eval_framework.tasks.base.Language attribute) TIS (eval_framework.tasks.base.Language attribute) TIT (eval_framework.tasks.base.Language attribute) TIU (eval_framework.tasks.base.Language attribute) TIV (eval_framework.tasks.base.Language attribute) TIW (eval_framework.tasks.base.Language attribute) TIX (eval_framework.tasks.base.Language attribute) TIY (eval_framework.tasks.base.Language attribute) TIZ (eval_framework.tasks.base.Language attribute) TJA (eval_framework.tasks.base.Language attribute) TJG (eval_framework.tasks.base.Language attribute) TJI (eval_framework.tasks.base.Language attribute) TJJ (eval_framework.tasks.base.Language attribute) TJL (eval_framework.tasks.base.Language attribute) TJM (eval_framework.tasks.base.Language attribute) TJN (eval_framework.tasks.base.Language attribute) TJO (eval_framework.tasks.base.Language attribute) TJP (eval_framework.tasks.base.Language attribute) TJS (eval_framework.tasks.base.Language attribute) TJU (eval_framework.tasks.base.Language attribute) TJW (eval_framework.tasks.base.Language attribute) TKA (eval_framework.tasks.base.Language attribute) TKB (eval_framework.tasks.base.Language attribute) TKD (eval_framework.tasks.base.Language attribute) TKE (eval_framework.tasks.base.Language attribute) TKF (eval_framework.tasks.base.Language attribute) TKG (eval_framework.tasks.base.Language attribute) TKK (eval_framework.tasks.base.Language attribute) TKL (eval_framework.tasks.base.Language attribute) TKM (eval_framework.tasks.base.Language attribute) TKN (eval_framework.tasks.base.Language attribute) TKP (eval_framework.tasks.base.Language attribute) TKQ (eval_framework.tasks.base.Language attribute) TKR (eval_framework.tasks.base.Language attribute) TKS (eval_framework.tasks.base.Language attribute) TKT (eval_framework.tasks.base.Language attribute) TKU (eval_framework.tasks.base.Language attribute) TKV (eval_framework.tasks.base.Language attribute) TKW (eval_framework.tasks.base.Language attribute) TKX (eval_framework.tasks.base.Language attribute) TKZ (eval_framework.tasks.base.Language attribute) TLA (eval_framework.tasks.base.Language attribute) TLB (eval_framework.tasks.base.Language attribute) TLC (eval_framework.tasks.base.Language attribute) TLD (eval_framework.tasks.base.Language attribute) TLE (eval_framework.tasks.base.Language attribute) TLF (eval_framework.tasks.base.Language attribute) TLG (eval_framework.tasks.base.Language attribute) TLH (eval_framework.tasks.base.Language attribute) TLI (eval_framework.tasks.base.Language attribute) TLJ (eval_framework.tasks.base.Language attribute) TLK (eval_framework.tasks.base.Language attribute) TLL (eval_framework.tasks.base.Language attribute) TLM (eval_framework.tasks.base.Language attribute) TLN (eval_framework.tasks.base.Language attribute) TLO (eval_framework.tasks.base.Language attribute) TLP (eval_framework.tasks.base.Language attribute) TLQ (eval_framework.tasks.base.Language attribute) TLR (eval_framework.tasks.base.Language attribute) TLS (eval_framework.tasks.base.Language attribute) TLT (eval_framework.tasks.base.Language attribute) TLU (eval_framework.tasks.base.Language attribute) TLV (eval_framework.tasks.base.Language attribute) TLW (eval_framework.tasks.base.Language attribute) TLX (eval_framework.tasks.base.Language attribute) TLY (eval_framework.tasks.base.Language attribute) TLZ (eval_framework.tasks.base.Language attribute) TMA (eval_framework.tasks.base.Language attribute) TMB (eval_framework.tasks.base.Language attribute) TMC (eval_framework.tasks.base.Language attribute) TMD (eval_framework.tasks.base.Language attribute) TME (eval_framework.tasks.base.Language attribute) TMF (eval_framework.tasks.base.Language attribute) TMG (eval_framework.tasks.base.Language attribute) TMH (eval_framework.tasks.base.Language attribute) TMI (eval_framework.tasks.base.Language attribute) TMJ (eval_framework.tasks.base.Language attribute) TMK (eval_framework.tasks.base.Language attribute) TML (eval_framework.tasks.base.Language attribute) TMM (eval_framework.tasks.base.Language attribute) TMN (eval_framework.tasks.base.Language attribute) TMO (eval_framework.tasks.base.Language attribute) TMP (eval_framework.tasks.base.Language attribute) TMQ (eval_framework.tasks.base.Language attribute) TMR (eval_framework.tasks.base.Language attribute) TMS (eval_framework.tasks.base.Language attribute) TMT (eval_framework.tasks.base.Language attribute) TMU (eval_framework.tasks.base.Language attribute) TMV (eval_framework.tasks.base.Language attribute) TMW (eval_framework.tasks.base.Language attribute) TMX (eval_framework.tasks.base.Language attribute) TMY (eval_framework.tasks.base.Language attribute) TMZ (eval_framework.tasks.base.Language attribute) TNA (eval_framework.tasks.base.Language attribute) TNB (eval_framework.tasks.base.Language attribute) TNC (eval_framework.tasks.base.Language attribute) TND (eval_framework.tasks.base.Language attribute) TNE (eval_framework.tasks.base.Language attribute) TNF (eval_framework.tasks.base.Language attribute) TNG (eval_framework.tasks.base.Language attribute) TNH (eval_framework.tasks.base.Language attribute) TNI (eval_framework.tasks.base.Language attribute) TNJ (eval_framework.tasks.base.Language attribute) TNK (eval_framework.tasks.base.Language attribute) TNL (eval_framework.tasks.base.Language attribute) TNM (eval_framework.tasks.base.Language attribute) TNN (eval_framework.tasks.base.Language attribute) TNO (eval_framework.tasks.base.Language attribute) TNP (eval_framework.tasks.base.Language attribute) TNQ (eval_framework.tasks.base.Language attribute) TNR (eval_framework.tasks.base.Language attribute) TNS (eval_framework.tasks.base.Language attribute) TNT (eval_framework.tasks.base.Language attribute) TNU (eval_framework.tasks.base.Language attribute) TNV (eval_framework.tasks.base.Language attribute) TNW (eval_framework.tasks.base.Language attribute) TNX (eval_framework.tasks.base.Language attribute) TNY (eval_framework.tasks.base.Language attribute) TNZ (eval_framework.tasks.base.Language attribute) TOB (eval_framework.tasks.base.Language attribute) TOC (eval_framework.tasks.base.Language attribute) TOD (eval_framework.tasks.base.Language attribute) TOE (eval_framework.tasks.base.Language attribute) TOF (eval_framework.tasks.base.Language attribute) TOG (eval_framework.tasks.base.Language attribute) TOH (eval_framework.tasks.base.Language attribute) TOI (eval_framework.tasks.base.Language attribute) TOJ (eval_framework.tasks.base.Language attribute) TOK (eval_framework.tasks.base.Language attribute) tokenize_path() (in module eval_framework.metrics.completion.struct_eval_metrics) TokenizedContainer (class in eval_framework.llm.vllm) tokenizer (eval_framework.llm.mistral.MistralVLLM property) (eval_framework.llm.vllm.BaseVLLMModel property) tokens (eval_framework.llm.vllm.TokenizedContainer attribute) TOL (eval_framework.tasks.base.Language attribute) TOM (eval_framework.tasks.base.Language attribute) TON (eval_framework.tasks.base.Language attribute) TOO (eval_framework.tasks.base.Language attribute) TOP (eval_framework.tasks.base.Language attribute) TOQ (eval_framework.tasks.base.Language attribute) TOR (eval_framework.tasks.base.Language attribute) TOS (eval_framework.tasks.base.Language attribute) TOT (eval_framework.tasks.base.Language attribute) TOU (eval_framework.tasks.base.Language attribute) TOV (eval_framework.tasks.base.Language attribute) TOW (eval_framework.tasks.base.Language attribute) TOX (eval_framework.tasks.base.Language attribute) TOY (eval_framework.tasks.base.Language attribute) TOZ (eval_framework.tasks.base.Language attribute) TPA (eval_framework.tasks.base.Language attribute) TPC (eval_framework.tasks.base.Language attribute) TPE (eval_framework.tasks.base.Language attribute) TPF (eval_framework.tasks.base.Language attribute) TPG (eval_framework.tasks.base.Language attribute) TPI (eval_framework.tasks.base.Language attribute) TPJ (eval_framework.tasks.base.Language attribute) TPK (eval_framework.tasks.base.Language attribute) TPL (eval_framework.tasks.base.Language attribute) TPM (eval_framework.tasks.base.Language attribute) TPN (eval_framework.tasks.base.Language attribute) TPO (eval_framework.tasks.base.Language attribute) TPP (eval_framework.tasks.base.Language attribute) TPQ (eval_framework.tasks.base.Language attribute) TPR (eval_framework.tasks.base.Language attribute) TPT (eval_framework.tasks.base.Language attribute) TPU (eval_framework.tasks.base.Language attribute) TPV (eval_framework.tasks.base.Language attribute) TPW (eval_framework.tasks.base.Language attribute) TPX (eval_framework.tasks.base.Language attribute) TPY (eval_framework.tasks.base.Language attribute) TPZ (eval_framework.tasks.base.Language attribute) TQB (eval_framework.tasks.base.Language attribute) TQL (eval_framework.tasks.base.Language attribute) TQM (eval_framework.tasks.base.Language attribute) TQN (eval_framework.tasks.base.Language attribute) TQO (eval_framework.tasks.base.Language attribute) TQP (eval_framework.tasks.base.Language attribute) TQQ (eval_framework.tasks.base.Language attribute) TQR (eval_framework.tasks.base.Language attribute) TQT (eval_framework.tasks.base.Language attribute) TQU (eval_framework.tasks.base.Language attribute) TQW (eval_framework.tasks.base.Language attribute) TRA (eval_framework.tasks.base.Language attribute) TRB (eval_framework.tasks.base.Language attribute) TRC (eval_framework.tasks.base.Language attribute) TRD (eval_framework.tasks.base.Language attribute) TRE (eval_framework.tasks.base.Language attribute) TRF (eval_framework.tasks.base.Language attribute) TRG (eval_framework.tasks.base.Language attribute) TRH (eval_framework.tasks.base.Language attribute) TRI (eval_framework.tasks.base.Language attribute) TRIVIAQA (class in eval_framework.tasks.benchmarks.triviaqa) TRJ (eval_framework.tasks.base.Language attribute) TRL (eval_framework.tasks.base.Language attribute) TRM (eval_framework.tasks.base.Language attribute) TRN (eval_framework.tasks.base.Language attribute) TRO (eval_framework.tasks.base.Language attribute) TRP (eval_framework.tasks.base.Language attribute) TRQ (eval_framework.tasks.base.Language attribute) TRR (eval_framework.tasks.base.Language attribute) TRS (eval_framework.tasks.base.Language attribute) TRT (eval_framework.tasks.base.Language attribute) TRU (eval_framework.tasks.base.Language attribute) TRUTHFULQA (class in eval_framework.tasks.benchmarks.truthfulqa) TRUTHFULQA_EU20_DE (class in eval_framework.tasks.benchmarks.opengptx_eu20) TRUTHFULQA_EU20_FR (class in eval_framework.tasks.benchmarks.opengptx_eu20) TRUTHFULQA_IDK (class in eval_framework.tasks.benchmarks.truthfulqa) TRV (eval_framework.tasks.base.Language attribute) TRW (eval_framework.tasks.base.Language attribute) TRX (eval_framework.tasks.base.Language attribute) TRY (eval_framework.tasks.base.Language attribute) TRZ (eval_framework.tasks.base.Language attribute) TSA (eval_framework.tasks.base.Language attribute) TSB (eval_framework.tasks.base.Language attribute) TSC (eval_framework.tasks.base.Language attribute) TSD (eval_framework.tasks.base.Language attribute) TSE (eval_framework.tasks.base.Language attribute) TSF (eval_framework.tasks.base.Language attribute) TSG (eval_framework.tasks.base.Language attribute) TSH (eval_framework.tasks.base.Language attribute) TSI (eval_framework.tasks.base.Language attribute) TSJ (eval_framework.tasks.base.Language attribute) TSK (eval_framework.tasks.base.Language attribute) TSL (eval_framework.tasks.base.Language attribute) TSM (eval_framework.tasks.base.Language attribute) TSN (eval_framework.tasks.base.Language attribute) TSO (eval_framework.tasks.base.Language attribute) TSP (eval_framework.tasks.base.Language attribute) TSQ (eval_framework.tasks.base.Language attribute) TSR (eval_framework.tasks.base.Language attribute) TSS (eval_framework.tasks.base.Language attribute) TST (eval_framework.tasks.base.Language attribute) TSU (eval_framework.tasks.base.Language attribute) TSV (eval_framework.tasks.base.Language attribute) TSW (eval_framework.tasks.base.Language attribute) TSX (eval_framework.tasks.base.Language attribute) TSY (eval_framework.tasks.base.Language attribute) TSZ (eval_framework.tasks.base.Language attribute) TTA (eval_framework.tasks.base.Language attribute) TTB (eval_framework.tasks.base.Language attribute) TTC (eval_framework.tasks.base.Language attribute) TTD (eval_framework.tasks.base.Language attribute) TTE (eval_framework.tasks.base.Language attribute) TTF (eval_framework.tasks.base.Language attribute) TTG (eval_framework.tasks.base.Language attribute) TTH (eval_framework.tasks.base.Language attribute) TTI (eval_framework.tasks.base.Language attribute) TTJ (eval_framework.tasks.base.Language attribute) TTK (eval_framework.tasks.base.Language attribute) TTL (eval_framework.tasks.base.Language attribute) TTM (eval_framework.tasks.base.Language attribute) TTN (eval_framework.tasks.base.Language attribute) TTO (eval_framework.tasks.base.Language attribute) TTP (eval_framework.tasks.base.Language attribute) TTQ (eval_framework.tasks.base.Language attribute) TTR (eval_framework.tasks.base.Language attribute) TTS (eval_framework.tasks.base.Language attribute) TTT (eval_framework.tasks.base.Language attribute) TTU (eval_framework.tasks.base.Language attribute) TTV (eval_framework.tasks.base.Language attribute) TTW (eval_framework.tasks.base.Language attribute) TTX (eval_framework.tasks.base.Language attribute) TTY (eval_framework.tasks.base.Language attribute) TTZ (eval_framework.tasks.base.Language attribute) TUA (eval_framework.tasks.base.Language attribute) TUB (eval_framework.tasks.base.Language attribute) TUC (eval_framework.tasks.base.Language attribute) TUD (eval_framework.tasks.base.Language attribute) TUE (eval_framework.tasks.base.Language attribute) TUF (eval_framework.tasks.base.Language attribute) TUG (eval_framework.tasks.base.Language attribute) TUH (eval_framework.tasks.base.Language attribute) TUI (eval_framework.tasks.base.Language attribute) TUJ (eval_framework.tasks.base.Language attribute) TUK (eval_framework.tasks.base.Language attribute) TUL (eval_framework.tasks.base.Language attribute) TUM (eval_framework.tasks.base.Language attribute) TUN (eval_framework.tasks.base.Language attribute) TUO (eval_framework.tasks.base.Language attribute) TUQ (eval_framework.tasks.base.Language attribute) TUR (eval_framework.tasks.base.Language attribute) TUS (eval_framework.tasks.base.Language attribute) TUU (eval_framework.tasks.base.Language attribute) TUV (eval_framework.tasks.base.Language attribute) TUX (eval_framework.tasks.base.Language attribute) TUY (eval_framework.tasks.base.Language attribute) TUZ (eval_framework.tasks.base.Language attribute) TVA (eval_framework.tasks.base.Language attribute) TVD (eval_framework.tasks.base.Language attribute) TVE (eval_framework.tasks.base.Language attribute) TVI (eval_framework.tasks.base.Language attribute) TVK (eval_framework.tasks.base.Language attribute) TVL (eval_framework.tasks.base.Language attribute) TVM (eval_framework.tasks.base.Language attribute) TVN (eval_framework.tasks.base.Language attribute) TVO (eval_framework.tasks.base.Language attribute) TVS (eval_framework.tasks.base.Language attribute) TVT (eval_framework.tasks.base.Language attribute) TVU (eval_framework.tasks.base.Language attribute) TVW (eval_framework.tasks.base.Language attribute) TVX (eval_framework.tasks.base.Language attribute) TVY (eval_framework.tasks.base.Language attribute) TWA (eval_framework.tasks.base.Language attribute) TWB (eval_framework.tasks.base.Language attribute) TWC (eval_framework.tasks.base.Language attribute) TWD (eval_framework.tasks.base.Language attribute) TWE (eval_framework.tasks.base.Language attribute) TWF (eval_framework.tasks.base.Language attribute) TWG (eval_framework.tasks.base.Language attribute) TWH (eval_framework.tasks.base.Language attribute) TWI (eval_framework.tasks.base.Language attribute) TWL (eval_framework.tasks.base.Language attribute) TWM (eval_framework.tasks.base.Language attribute) TWN (eval_framework.tasks.base.Language attribute) TWO (eval_framework.tasks.base.Language attribute) TWP (eval_framework.tasks.base.Language attribute) TWQ (eval_framework.tasks.base.Language attribute) TWR (eval_framework.tasks.base.Language attribute) TWT (eval_framework.tasks.base.Language attribute) TWU (eval_framework.tasks.base.Language attribute) TWW (eval_framework.tasks.base.Language attribute) TWX (eval_framework.tasks.base.Language attribute) TWY (eval_framework.tasks.base.Language attribute) TXA (eval_framework.tasks.base.Language attribute) TXB (eval_framework.tasks.base.Language attribute) TXC (eval_framework.tasks.base.Language attribute) TXE (eval_framework.tasks.base.Language attribute) TXG (eval_framework.tasks.base.Language attribute) TXH (eval_framework.tasks.base.Language attribute) TXI (eval_framework.tasks.base.Language attribute) TXJ (eval_framework.tasks.base.Language attribute) TXM (eval_framework.tasks.base.Language attribute) TXN (eval_framework.tasks.base.Language attribute) TXO (eval_framework.tasks.base.Language attribute) TXQ (eval_framework.tasks.base.Language attribute) TXR (eval_framework.tasks.base.Language attribute) TXS (eval_framework.tasks.base.Language attribute) TXT (eval_framework.tasks.base.Language attribute) TXU (eval_framework.tasks.base.Language attribute) TXX (eval_framework.tasks.base.Language attribute) TXY (eval_framework.tasks.base.Language attribute) TYA (eval_framework.tasks.base.Language attribute) TYE (eval_framework.tasks.base.Language attribute) TYH (eval_framework.tasks.base.Language attribute) TYI (eval_framework.tasks.base.Language attribute) TYJ (eval_framework.tasks.base.Language attribute) TYL (eval_framework.tasks.base.Language attribute) TYN (eval_framework.tasks.base.Language attribute) TYP (eval_framework.tasks.base.Language attribute) type (eval_framework.tasks.perturbation.PerturbationConfig attribute) TYR (eval_framework.tasks.base.Language attribute) TYS (eval_framework.tasks.base.Language attribute) TYT (eval_framework.tasks.base.Language attribute) TYU (eval_framework.tasks.base.Language attribute) TYV (eval_framework.tasks.base.Language attribute) TYX (eval_framework.tasks.base.Language attribute) TYY (eval_framework.tasks.base.Language attribute) TYZ (eval_framework.tasks.base.Language attribute) TZA (eval_framework.tasks.base.Language attribute) TZB (eval_framework.tasks.base.Language attribute) TZC (eval_framework.tasks.base.Language attribute) TZE (eval_framework.tasks.base.Language attribute) TZH (eval_framework.tasks.base.Language attribute) TZJ (eval_framework.tasks.base.Language attribute) TZL (eval_framework.tasks.base.Language attribute) TZM (eval_framework.tasks.base.Language attribute) TZN (eval_framework.tasks.base.Language attribute) TZO (eval_framework.tasks.base.Language attribute) TZS (eval_framework.tasks.base.Language attribute) TZT (eval_framework.tasks.base.Language attribute) TZU (eval_framework.tasks.base.Language attribute) TZX (eval_framework.tasks.base.Language attribute) TZZ (eval_framework.tasks.base.Language attribute) U UAM (eval_framework.tasks.base.Language attribute) UAN (eval_framework.tasks.base.Language attribute) UAR (eval_framework.tasks.base.Language attribute) UBA (eval_framework.tasks.base.Language attribute) UBI (eval_framework.tasks.base.Language attribute) UBL (eval_framework.tasks.base.Language attribute) UBM (eval_framework.tasks.base.Language attribute) UBR (eval_framework.tasks.base.Language attribute) UBU (eval_framework.tasks.base.Language attribute) UBY (eval_framework.tasks.base.Language attribute) UDA (eval_framework.tasks.base.Language attribute) UDE (eval_framework.tasks.base.Language attribute) UDG (eval_framework.tasks.base.Language attribute) UDI (eval_framework.tasks.base.Language attribute) UDJ (eval_framework.tasks.base.Language attribute) UDL (eval_framework.tasks.base.Language attribute) UDM (eval_framework.tasks.base.Language attribute) UDU (eval_framework.tasks.base.Language attribute) UES (eval_framework.tasks.base.Language attribute) UFI (eval_framework.tasks.base.Language attribute) UGA (eval_framework.tasks.base.Language attribute) UGB (eval_framework.tasks.base.Language attribute) UGE (eval_framework.tasks.base.Language attribute) UGH (eval_framework.tasks.base.Language attribute) UGN (eval_framework.tasks.base.Language attribute) UGO (eval_framework.tasks.base.Language attribute) UGY (eval_framework.tasks.base.Language attribute) UHA (eval_framework.tasks.base.Language attribute) UHN (eval_framework.tasks.base.Language attribute) UIG (eval_framework.tasks.base.Language attribute) UIS (eval_framework.tasks.base.Language attribute) UIV (eval_framework.tasks.base.Language attribute) UJI (eval_framework.tasks.base.Language attribute) UKA (eval_framework.tasks.base.Language attribute) UKG (eval_framework.tasks.base.Language attribute) UKH (eval_framework.tasks.base.Language attribute) UKI (eval_framework.tasks.base.Language attribute) UKK (eval_framework.tasks.base.Language attribute) UKL (eval_framework.tasks.base.Language attribute) UKP (eval_framework.tasks.base.Language attribute) UKQ (eval_framework.tasks.base.Language attribute) UKR (eval_framework.tasks.base.Language attribute) UKS (eval_framework.tasks.base.Language attribute) UKU (eval_framework.tasks.base.Language attribute) UKV (eval_framework.tasks.base.Language attribute) UKW (eval_framework.tasks.base.Language attribute) UKY (eval_framework.tasks.base.Language attribute) ULA (eval_framework.tasks.base.Language attribute) ULB (eval_framework.tasks.base.Language attribute) ULC (eval_framework.tasks.base.Language attribute) ULE (eval_framework.tasks.base.Language attribute) ULF (eval_framework.tasks.base.Language attribute) ULI (eval_framework.tasks.base.Language attribute) ULK (eval_framework.tasks.base.Language attribute) ULL (eval_framework.tasks.base.Language attribute) ULM (eval_framework.tasks.base.Language attribute) ULN (eval_framework.tasks.base.Language attribute) ULU (eval_framework.tasks.base.Language attribute) ULW (eval_framework.tasks.base.Language attribute) ULY (eval_framework.tasks.base.Language attribute) UMA (eval_framework.tasks.base.Language attribute) UMB (eval_framework.tasks.base.Language attribute) UMC (eval_framework.tasks.base.Language attribute) UMD (eval_framework.tasks.base.Language attribute) UMG (eval_framework.tasks.base.Language attribute) UMI (eval_framework.tasks.base.Language attribute) UMM (eval_framework.tasks.base.Language attribute) UMN (eval_framework.tasks.base.Language attribute) UMO (eval_framework.tasks.base.Language attribute) UMP (eval_framework.tasks.base.Language attribute) UMR (eval_framework.tasks.base.Language attribute) UMS (eval_framework.tasks.base.Language attribute) UMU (eval_framework.tasks.base.Language attribute) UNA (eval_framework.tasks.base.Language attribute) UNANSWERABLE_STR (eval_framework.tasks.benchmarks.squad.SQUAD2 attribute) UND (eval_framework.tasks.base.Language attribute) UNE (eval_framework.tasks.base.Language attribute) UNG (eval_framework.tasks.base.Language attribute) UNI (eval_framework.tasks.base.Language attribute) unittest_merge_snippets() (in module eval_framework.tasks.utils) UNK (eval_framework.tasks.base.Language attribute) UNM (eval_framework.tasks.base.Language attribute) UNN (eval_framework.tasks.base.Language attribute) UNP (eval_framework.tasks.base.Language attribute) UNR (eval_framework.tasks.base.Language attribute) UNU (eval_framework.tasks.base.Language attribute) UNX (eval_framework.tasks.base.Language attribute) UNZ (eval_framework.tasks.base.Language attribute) UOK (eval_framework.tasks.base.Language attribute) UON (eval_framework.tasks.base.Language attribute) update_changed_datasets_only() (in module eval_framework.tasks.task_names) UPI (eval_framework.tasks.base.Language attribute) upload() (eval_framework.result_processors.base.ResultsUploader method) (eval_framework.result_processors.hf_uploader.HFUploader method) (eval_framework.result_processors.wandb_uploader.WandbUploader method) upper_case_string() (eval_framework.tasks.utils.HatPaperEditor method) UPPERCASE (eval_framework.tasks.perturbation.PerturbationType attribute) UPV (eval_framework.tasks.base.Language attribute) URA (eval_framework.tasks.base.Language attribute) URB (eval_framework.tasks.base.Language attribute) URC (eval_framework.tasks.base.Language attribute) URD (eval_framework.tasks.base.Language attribute) URE (eval_framework.tasks.base.Language attribute) URF (eval_framework.tasks.base.Language attribute) URG (eval_framework.tasks.base.Language attribute) URH (eval_framework.tasks.base.Language attribute) URI (eval_framework.tasks.base.Language attribute) URK (eval_framework.tasks.base.Language attribute) URL (eval_framework.tasks.base.Language attribute) URM (eval_framework.tasks.base.Language attribute) URN (eval_framework.tasks.base.Language attribute) URO (eval_framework.tasks.base.Language attribute) URP (eval_framework.tasks.base.Language attribute) URR (eval_framework.tasks.base.Language attribute) URT (eval_framework.tasks.base.Language attribute) URU (eval_framework.tasks.base.Language attribute) URV (eval_framework.tasks.base.Language attribute) URW (eval_framework.tasks.base.Language attribute) URX (eval_framework.tasks.base.Language attribute) URY (eval_framework.tasks.base.Language attribute) URZ (eval_framework.tasks.base.Language attribute) USA (eval_framework.tasks.base.Language attribute) USH (eval_framework.tasks.base.Language attribute) USI (eval_framework.tasks.base.Language attribute) USK (eval_framework.tasks.base.Language attribute) USP (eval_framework.tasks.base.Language attribute) USS (eval_framework.tasks.base.Language attribute) USU (eval_framework.tasks.base.Language attribute) UTA (eval_framework.tasks.base.Language attribute) UTE (eval_framework.tasks.base.Language attribute) UTH (eval_framework.tasks.base.Language attribute) UTP (eval_framework.tasks.base.Language attribute) UTR (eval_framework.tasks.base.Language attribute) UTU (eval_framework.tasks.base.Language attribute) UUM (eval_framework.tasks.base.Language attribute) UUN (eval_framework.tasks.base.Language attribute) UUR (eval_framework.tasks.base.Language attribute) UUU (eval_framework.tasks.base.Language attribute) UVE (eval_framework.tasks.base.Language attribute) UVH (eval_framework.tasks.base.Language attribute) UVL (eval_framework.tasks.base.Language attribute) UWA (eval_framework.tasks.base.Language attribute) UYA (eval_framework.tasks.base.Language attribute) UZB (eval_framework.tasks.base.Language attribute) UZN (eval_framework.tasks.base.Language attribute) UZS (eval_framework.tasks.base.Language attribute) V VAA (eval_framework.tasks.base.Language attribute) VAE (eval_framework.tasks.base.Language attribute) VAF (eval_framework.tasks.base.Language attribute) VAG (eval_framework.tasks.base.Language attribute) VAH (eval_framework.tasks.base.Language attribute) VAI (eval_framework.tasks.base.Language attribute) VAJ (eval_framework.tasks.base.Language attribute) VAL (eval_framework.tasks.base.Language attribute) validate_judge_model_args() (eval_framework.tasks.eval_config.EvalConfig class method) validate_llm_args() (eval_framework.tasks.eval_config.EvalConfig class method) validate_llm_judge_defined() (eval_framework.tasks.eval_config.EvalConfig method) validate_output_dir() (eval_framework.tasks.eval_config.EvalConfig class method) validate_query() (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) validate_query_mysql() (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) validate_query_postgres() (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) validate_query_sqlite() (eval_framework.metrics.llm.llm_judge_sql.LLMJudgeSql method) validate_task_name() (in module eval_framework.tasks.registry) value (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) (eval_framework.tasks.task_names.TaskNameEnum property) VAM (eval_framework.tasks.base.Language attribute) VAN (eval_framework.tasks.base.Language attribute) VAO (eval_framework.tasks.base.Language attribute) VAP (eval_framework.tasks.base.Language attribute) VAR (eval_framework.tasks.base.Language attribute) VAS (eval_framework.tasks.base.Language attribute) VAU (eval_framework.tasks.base.Language attribute) VAV (eval_framework.tasks.base.Language attribute) VAY (eval_framework.tasks.base.Language attribute) VBB (eval_framework.tasks.base.Language attribute) VBK (eval_framework.tasks.base.Language attribute) VEC (eval_framework.tasks.base.Language attribute) VED (eval_framework.tasks.base.Language attribute) VEL (eval_framework.tasks.base.Language attribute) VEM (eval_framework.tasks.base.Language attribute) VEN (eval_framework.tasks.base.Language attribute) VEO (eval_framework.tasks.base.Language attribute) VEP (eval_framework.tasks.base.Language attribute) VER (eval_framework.tasks.base.Language attribute) verbose (eval_framework.tasks.perturbation.PerturbationConfig attribute) VGR (eval_framework.tasks.base.Language attribute) VGT (eval_framework.tasks.base.Language attribute) VIC (eval_framework.tasks.base.Language attribute) VID (eval_framework.tasks.base.Language attribute) VIE (eval_framework.tasks.base.Language attribute) VIF (eval_framework.tasks.base.Language attribute) VIG (eval_framework.tasks.base.Language attribute) VIL (eval_framework.tasks.base.Language attribute) VIN (eval_framework.tasks.base.Language attribute) VIS (eval_framework.tasks.base.Language attribute) VIT (eval_framework.tasks.base.Language attribute) VIV (eval_framework.tasks.base.Language attribute) VJK (eval_framework.tasks.base.Language attribute) VKA (eval_framework.tasks.base.Language attribute) VKI (eval_framework.tasks.base.Language attribute) VKJ (eval_framework.tasks.base.Language attribute) VKK (eval_framework.tasks.base.Language attribute) VKL (eval_framework.tasks.base.Language attribute) VKM (eval_framework.tasks.base.Language attribute) VKN (eval_framework.tasks.base.Language attribute) VKO (eval_framework.tasks.base.Language attribute) VKP (eval_framework.tasks.base.Language attribute) VKT (eval_framework.tasks.base.Language attribute) VKU (eval_framework.tasks.base.Language attribute) VKY (eval_framework.tasks.base.Language attribute) VKZ (eval_framework.tasks.base.Language attribute) VLLMModel (class in eval_framework.llm.vllm) VLLMRegistryModel (class in eval_framework.llm.vllm) VLLMTokenizer (class in eval_framework.llm.vllm) VLLMTokenizerAPI (class in eval_framework.llm.vllm) VLP (eval_framework.tasks.base.Language attribute) VLR (eval_framework.tasks.base.Language attribute) VLS (eval_framework.tasks.base.Language attribute) VMA (eval_framework.tasks.base.Language attribute) VMB (eval_framework.tasks.base.Language attribute) VMC (eval_framework.tasks.base.Language attribute) VMD (eval_framework.tasks.base.Language attribute) VME (eval_framework.tasks.base.Language attribute) VMF (eval_framework.tasks.base.Language attribute) VMG (eval_framework.tasks.base.Language attribute) VMH (eval_framework.tasks.base.Language attribute) VMI (eval_framework.tasks.base.Language attribute) VMJ (eval_framework.tasks.base.Language attribute) VMK (eval_framework.tasks.base.Language attribute) VML (eval_framework.tasks.base.Language attribute) VMM (eval_framework.tasks.base.Language attribute) VMO (eval_framework.tasks.base.Language attribute) VMP (eval_framework.tasks.base.Language attribute) VMQ (eval_framework.tasks.base.Language attribute) VMR (eval_framework.tasks.base.Language attribute) VMS (eval_framework.tasks.base.Language attribute) VMU (eval_framework.tasks.base.Language attribute) VMV (eval_framework.tasks.base.Language attribute) VMW (eval_framework.tasks.base.Language attribute) VMX (eval_framework.tasks.base.Language attribute) VMY (eval_framework.tasks.base.Language attribute) VMZ (eval_framework.tasks.base.Language attribute) VNK (eval_framework.tasks.base.Language attribute) VNM (eval_framework.tasks.base.Language attribute) VNP (eval_framework.tasks.base.Language attribute) VOL (eval_framework.tasks.base.Language attribute) VOR (eval_framework.tasks.base.Language attribute) VOT (eval_framework.tasks.base.Language attribute) VRA (eval_framework.tasks.base.Language attribute) VRO (eval_framework.tasks.base.Language attribute) VRS (eval_framework.tasks.base.Language attribute) VRT (eval_framework.tasks.base.Language attribute) VSI (eval_framework.tasks.base.Language attribute) VSL (eval_framework.tasks.base.Language attribute) VSN (eval_framework.tasks.base.Language attribute) VSV (eval_framework.tasks.base.Language attribute) VTO (eval_framework.tasks.base.Language attribute) VUM (eval_framework.tasks.base.Language attribute) VUN (eval_framework.tasks.base.Language attribute) VUT (eval_framework.tasks.base.Language attribute) VWA (eval_framework.tasks.base.Language attribute) W WAA (eval_framework.tasks.base.Language attribute) WAB (eval_framework.tasks.base.Language attribute) WAC (eval_framework.tasks.base.Language attribute) WAD (eval_framework.tasks.base.Language attribute) WAE (eval_framework.tasks.base.Language attribute) WAF (eval_framework.tasks.base.Language attribute) WAG (eval_framework.tasks.base.Language attribute) WAH (eval_framework.tasks.base.Language attribute) WAI (eval_framework.tasks.base.Language attribute) WAJ (eval_framework.tasks.base.Language attribute) WAL (eval_framework.tasks.base.Language attribute) WAM (eval_framework.tasks.base.Language attribute) WAN (eval_framework.tasks.base.Language attribute) wandb_entity (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_project (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_run_id (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_upload_results (eval_framework.context.determined.Hyperparameters attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) WandbUploader (class in eval_framework.result_processors.wandb_uploader) WAO (eval_framework.tasks.base.Language attribute) WAP (eval_framework.tasks.base.Language attribute) WAQ (eval_framework.tasks.base.Language attribute) WAR (eval_framework.tasks.base.Language attribute) WAS (eval_framework.tasks.base.Language attribute) WAT (eval_framework.tasks.base.Language attribute) WAU (eval_framework.tasks.base.Language attribute) WAV (eval_framework.tasks.base.Language attribute) WAW (eval_framework.tasks.base.Language attribute) WAX (eval_framework.tasks.base.Language attribute) WAY (eval_framework.tasks.base.Language attribute) WAZ (eval_framework.tasks.base.Language attribute) WBA (eval_framework.tasks.base.Language attribute) WBB (eval_framework.tasks.base.Language attribute) WBE (eval_framework.tasks.base.Language attribute) WBF (eval_framework.tasks.base.Language attribute) WBH (eval_framework.tasks.base.Language attribute) WBI (eval_framework.tasks.base.Language attribute) WBJ (eval_framework.tasks.base.Language attribute) WBK (eval_framework.tasks.base.Language attribute) WBL (eval_framework.tasks.base.Language attribute) WBM (eval_framework.tasks.base.Language attribute) WBP (eval_framework.tasks.base.Language attribute) WBQ (eval_framework.tasks.base.Language attribute) WBR (eval_framework.tasks.base.Language attribute) WBS (eval_framework.tasks.base.Language attribute) WBT (eval_framework.tasks.base.Language attribute) WBV (eval_framework.tasks.base.Language attribute) WBW (eval_framework.tasks.base.Language attribute) WCA (eval_framework.tasks.base.Language attribute) WCI (eval_framework.tasks.base.Language attribute) WDD (eval_framework.tasks.base.Language attribute) WDG (eval_framework.tasks.base.Language attribute) WDJ (eval_framework.tasks.base.Language attribute) WDK (eval_framework.tasks.base.Language attribute) WDT (eval_framework.tasks.base.Language attribute) WDU (eval_framework.tasks.base.Language attribute) WDY (eval_framework.tasks.base.Language attribute) WEA (eval_framework.tasks.base.Language attribute) WEC (eval_framework.tasks.base.Language attribute) WED (eval_framework.tasks.base.Language attribute) WEG (eval_framework.tasks.base.Language attribute) WEH (eval_framework.tasks.base.Language attribute) WEI (eval_framework.tasks.base.Language attribute) WEM (eval_framework.tasks.base.Language attribute) WEO (eval_framework.tasks.base.Language attribute) WEP (eval_framework.tasks.base.Language attribute) WER (eval_framework.tasks.base.Language attribute) WES (eval_framework.tasks.base.Language attribute) WET (eval_framework.tasks.base.Language attribute) WEU (eval_framework.tasks.base.Language attribute) WEW (eval_framework.tasks.base.Language attribute) WFG (eval_framework.tasks.base.Language attribute) WGA (eval_framework.tasks.base.Language attribute) WGB (eval_framework.tasks.base.Language attribute) WGG (eval_framework.tasks.base.Language attribute) WGI (eval_framework.tasks.base.Language attribute) WGO (eval_framework.tasks.base.Language attribute) WGU (eval_framework.tasks.base.Language attribute) WGW (eval_framework.tasks.base.Language attribute) WGY (eval_framework.tasks.base.Language attribute) WHA (eval_framework.tasks.base.Language attribute) WHG (eval_framework.tasks.base.Language attribute) WHK (eval_framework.tasks.base.Language attribute) WHU (eval_framework.tasks.base.Language attribute) WIB (eval_framework.tasks.base.Language attribute) WIC (eval_framework.tasks.base.Language attribute) WIE (eval_framework.tasks.base.Language attribute) WIF (eval_framework.tasks.base.Language attribute) WIG (eval_framework.tasks.base.Language attribute) WIH (eval_framework.tasks.base.Language attribute) WII (eval_framework.tasks.base.Language attribute) WIJ (eval_framework.tasks.base.Language attribute) WIK (eval_framework.tasks.base.Language attribute) WIL (eval_framework.tasks.base.Language attribute) WIM (eval_framework.tasks.base.Language attribute) WIN (eval_framework.tasks.base.Language attribute) WINOGENDER (class in eval_framework.tasks.benchmarks.winogender) WINOGENDER_IDK (class in eval_framework.tasks.benchmarks.winogender) WINOGRANDE (class in eval_framework.tasks.benchmarks.winogrande) WINOGRANDE_IDK (class in eval_framework.tasks.benchmarks.winogrande) WINOX (class in eval_framework.tasks.benchmarks.winox) WINOX_DE (class in eval_framework.tasks.benchmarks.winox) WINOX_FR (class in eval_framework.tasks.benchmarks.winox) WIR (eval_framework.tasks.base.Language attribute) WIT (eval_framework.tasks.base.Language attribute) with_overwrite() (eval_framework.tasks.base.BaseTask class method) with_registry() (in module eval_framework.tasks.registry) WIU (eval_framework.tasks.base.Language attribute) WIV (eval_framework.tasks.base.Language attribute) WIW (eval_framework.tasks.base.Language attribute) WIY (eval_framework.tasks.base.Language attribute) WJA (eval_framework.tasks.base.Language attribute) WJI (eval_framework.tasks.base.Language attribute) WKA (eval_framework.tasks.base.Language attribute) WKB (eval_framework.tasks.base.Language attribute) WKD (eval_framework.tasks.base.Language attribute) WKL (eval_framework.tasks.base.Language attribute) WKR (eval_framework.tasks.base.Language attribute) WKU (eval_framework.tasks.base.Language attribute) WKW (eval_framework.tasks.base.Language attribute) WKY (eval_framework.tasks.base.Language attribute) WLA (eval_framework.tasks.base.Language attribute) WLC (eval_framework.tasks.base.Language attribute) WLE (eval_framework.tasks.base.Language attribute) WLG (eval_framework.tasks.base.Language attribute) WLH (eval_framework.tasks.base.Language attribute) WLI (eval_framework.tasks.base.Language attribute) WLK (eval_framework.tasks.base.Language attribute) WLL (eval_framework.tasks.base.Language attribute) WLM (eval_framework.tasks.base.Language attribute) WLN (eval_framework.tasks.base.Language attribute) WLO (eval_framework.tasks.base.Language attribute) WLR (eval_framework.tasks.base.Language attribute) WLS (eval_framework.tasks.base.Language attribute) WLU (eval_framework.tasks.base.Language attribute) WLV (eval_framework.tasks.base.Language attribute) WLW (eval_framework.tasks.base.Language attribute) WLX (eval_framework.tasks.base.Language attribute) WLY (eval_framework.tasks.base.Language attribute) WMA (eval_framework.tasks.base.Language attribute) WMB (eval_framework.tasks.base.Language attribute) WMC (eval_framework.tasks.base.Language attribute) WMD (eval_framework.tasks.base.Language attribute) WME (eval_framework.tasks.base.Language attribute) WMG (eval_framework.tasks.base.Language attribute) WMH (eval_framework.tasks.base.Language attribute) WMI (eval_framework.tasks.base.Language attribute) WMM (eval_framework.tasks.base.Language attribute) WMN (eval_framework.tasks.base.Language attribute) WMO (eval_framework.tasks.base.Language attribute) WMS (eval_framework.tasks.base.Language attribute) WMT (class in eval_framework.tasks.benchmarks.wmt) (eval_framework.tasks.base.Language attribute) WMT14 (class in eval_framework.tasks.benchmarks.wmt) WMT14_INSTRUCT (class in eval_framework.tasks.benchmarks.wmt) WMT16 (class in eval_framework.tasks.benchmarks.wmt) WMT16_INSTRUCT (class in eval_framework.tasks.benchmarks.wmt) WMT20 (class in eval_framework.tasks.benchmarks.wmt) WMT20_INSTRUCT (class in eval_framework.tasks.benchmarks.wmt) WMT_INSTRUCT (class in eval_framework.tasks.benchmarks.wmt) WMW (eval_framework.tasks.base.Language attribute) WMX (eval_framework.tasks.base.Language attribute) WNB (eval_framework.tasks.base.Language attribute) WNC (eval_framework.tasks.base.Language attribute) WND (eval_framework.tasks.base.Language attribute) WNE (eval_framework.tasks.base.Language attribute) WNG (eval_framework.tasks.base.Language attribute) WNI (eval_framework.tasks.base.Language attribute) WNK (eval_framework.tasks.base.Language attribute) WNM (eval_framework.tasks.base.Language attribute) WNN (eval_framework.tasks.base.Language attribute) WNO (eval_framework.tasks.base.Language attribute) WNP (eval_framework.tasks.base.Language attribute) WNU (eval_framework.tasks.base.Language attribute) WNW (eval_framework.tasks.base.Language attribute) WNY (eval_framework.tasks.base.Language attribute) WOA (eval_framework.tasks.base.Language attribute) WOB (eval_framework.tasks.base.Language attribute) WOC (eval_framework.tasks.base.Language attribute) WOD (eval_framework.tasks.base.Language attribute) WOE (eval_framework.tasks.base.Language attribute) WOF (eval_framework.tasks.base.Language attribute) WOG (eval_framework.tasks.base.Language attribute) WOI (eval_framework.tasks.base.Language attribute) WOK (eval_framework.tasks.base.Language attribute) WOL (eval_framework.tasks.base.Language attribute) WOM (eval_framework.tasks.base.Language attribute) WON (eval_framework.tasks.base.Language attribute) WOO (eval_framework.tasks.base.Language attribute) WOR (eval_framework.tasks.base.Language attribute) word_count (eval_framework.metrics.completion.text_counter.WordCounterMetricContext attribute) WordCounter (class in eval_framework.metrics.completion.text_counter) WordCounterMetricContext (class in eval_framework.metrics.completion.text_counter) WordRepetition (class in eval_framework.metrics.completion.repetition) WORDS (eval_framework.metrics.completion.length_control.LengthRequirementUnit attribute) WOS (eval_framework.tasks.base.Language attribute) WOW (eval_framework.tasks.base.Language attribute) WOY (eval_framework.tasks.base.Language attribute) WPC (eval_framework.tasks.base.Language attribute) WRA (eval_framework.tasks.base.Language attribute) WRB (eval_framework.tasks.base.Language attribute) WRD (eval_framework.tasks.base.Language attribute) WRE (eval_framework.tasks.base.Language attribute) WRG (eval_framework.tasks.base.Language attribute) WRH (eval_framework.tasks.base.Language attribute) WRI (eval_framework.tasks.base.Language attribute) WRK (eval_framework.tasks.base.Language attribute) WRL (eval_framework.tasks.base.Language attribute) WRM (eval_framework.tasks.base.Language attribute) WRN (eval_framework.tasks.base.Language attribute) WRO (eval_framework.tasks.base.Language attribute) WRP (eval_framework.tasks.base.Language attribute) WRR (eval_framework.tasks.base.Language attribute) WRS (eval_framework.tasks.base.Language attribute) WRU (eval_framework.tasks.base.Language attribute) WRV (eval_framework.tasks.base.Language attribute) WRW (eval_framework.tasks.base.Language attribute) WRX (eval_framework.tasks.base.Language attribute) WRY (eval_framework.tasks.base.Language attribute) WRZ (eval_framework.tasks.base.Language attribute) WSA (eval_framework.tasks.base.Language attribute) WSG (eval_framework.tasks.base.Language attribute) WSI (eval_framework.tasks.base.Language attribute) WSK (eval_framework.tasks.base.Language attribute) WSR (eval_framework.tasks.base.Language attribute) WSS (eval_framework.tasks.base.Language attribute) WSU (eval_framework.tasks.base.Language attribute) WSV (eval_framework.tasks.base.Language attribute) WTB (eval_framework.tasks.base.Language attribute) WTF (eval_framework.tasks.base.Language attribute) WTH (eval_framework.tasks.base.Language attribute) WTI (eval_framework.tasks.base.Language attribute) WTK (eval_framework.tasks.base.Language attribute) WTM (eval_framework.tasks.base.Language attribute) WTW (eval_framework.tasks.base.Language attribute) WUA (eval_framework.tasks.base.Language attribute) WUB (eval_framework.tasks.base.Language attribute) WUD (eval_framework.tasks.base.Language attribute) WUH (eval_framework.tasks.base.Language attribute) WUL (eval_framework.tasks.base.Language attribute) WUM (eval_framework.tasks.base.Language attribute) WUN (eval_framework.tasks.base.Language attribute) WUR (eval_framework.tasks.base.Language attribute) WUT (eval_framework.tasks.base.Language attribute) WUU (eval_framework.tasks.base.Language attribute) WUV (eval_framework.tasks.base.Language attribute) WUX (eval_framework.tasks.base.Language attribute) WUY (eval_framework.tasks.base.Language attribute) WWA (eval_framework.tasks.base.Language attribute) WWB (eval_framework.tasks.base.Language attribute) WWO (eval_framework.tasks.base.Language attribute) WWR (eval_framework.tasks.base.Language attribute) WWW (eval_framework.tasks.base.Language attribute) WXA (eval_framework.tasks.base.Language attribute) WXW (eval_framework.tasks.base.Language attribute) WYA (eval_framework.tasks.base.Language attribute) WYB (eval_framework.tasks.base.Language attribute) WYI (eval_framework.tasks.base.Language attribute) WYM (eval_framework.tasks.base.Language attribute) WYN (eval_framework.tasks.base.Language attribute) WYR (eval_framework.tasks.base.Language attribute) WYY (eval_framework.tasks.base.Language attribute) X XAA (eval_framework.tasks.base.Language attribute) XAB (eval_framework.tasks.base.Language attribute) XAC (eval_framework.tasks.base.Language attribute) XAD (eval_framework.tasks.base.Language attribute) XAE (eval_framework.tasks.base.Language attribute) XAG (eval_framework.tasks.base.Language attribute) XAH (eval_framework.tasks.base.Language attribute) XAI (eval_framework.tasks.base.Language attribute) XAJ (eval_framework.tasks.base.Language attribute) XAK (eval_framework.tasks.base.Language attribute) XAL (eval_framework.tasks.base.Language attribute) XAM (eval_framework.tasks.base.Language attribute) XAN (eval_framework.tasks.base.Language attribute) XAO (eval_framework.tasks.base.Language attribute) XAP (eval_framework.tasks.base.Language attribute) XAQ (eval_framework.tasks.base.Language attribute) XAR (eval_framework.tasks.base.Language attribute) XAS (eval_framework.tasks.base.Language attribute) XAT (eval_framework.tasks.base.Language attribute) XAU (eval_framework.tasks.base.Language attribute) XAV (eval_framework.tasks.base.Language attribute) XAW (eval_framework.tasks.base.Language attribute) XAY (eval_framework.tasks.base.Language attribute) XBA (eval_framework.tasks.base.Language attribute) XBB (eval_framework.tasks.base.Language attribute) XBC (eval_framework.tasks.base.Language attribute) XBD (eval_framework.tasks.base.Language attribute) XBE (eval_framework.tasks.base.Language attribute) XBG (eval_framework.tasks.base.Language attribute) XBI (eval_framework.tasks.base.Language attribute) XBJ (eval_framework.tasks.base.Language attribute) XBM (eval_framework.tasks.base.Language attribute) XBN (eval_framework.tasks.base.Language attribute) XBO (eval_framework.tasks.base.Language attribute) XBP (eval_framework.tasks.base.Language attribute) XBR (eval_framework.tasks.base.Language attribute) XBW (eval_framework.tasks.base.Language attribute) XBX (eval_framework.tasks.base.Language attribute) XBY (eval_framework.tasks.base.Language attribute) XCB (eval_framework.tasks.base.Language attribute) XCC (eval_framework.tasks.base.Language attribute) XCE (eval_framework.tasks.base.Language attribute) XCG (eval_framework.tasks.base.Language attribute) XCH (eval_framework.tasks.base.Language attribute) XCL (eval_framework.tasks.base.Language attribute) XCM (eval_framework.tasks.base.Language attribute) XCN (eval_framework.tasks.base.Language attribute) XCO (eval_framework.tasks.base.Language attribute) XCR (eval_framework.tasks.base.Language attribute) XCT (eval_framework.tasks.base.Language attribute) XCU (eval_framework.tasks.base.Language attribute) XCV (eval_framework.tasks.base.Language attribute) XCW (eval_framework.tasks.base.Language attribute) XCY (eval_framework.tasks.base.Language attribute) XDA (eval_framework.tasks.base.Language attribute) XDC (eval_framework.tasks.base.Language attribute) XDK (eval_framework.tasks.base.Language attribute) XDM (eval_framework.tasks.base.Language attribute) XDO (eval_framework.tasks.base.Language attribute) XDQ (eval_framework.tasks.base.Language attribute) XDY (eval_framework.tasks.base.Language attribute) XEB (eval_framework.tasks.base.Language attribute) XED (eval_framework.tasks.base.Language attribute) XEG (eval_framework.tasks.base.Language attribute) XEL (eval_framework.tasks.base.Language attribute) XEM (eval_framework.tasks.base.Language attribute) XEP (eval_framework.tasks.base.Language attribute) XER (eval_framework.tasks.base.Language attribute) XES (eval_framework.tasks.base.Language attribute) XET (eval_framework.tasks.base.Language attribute) XEU (eval_framework.tasks.base.Language attribute) XFA (eval_framework.tasks.base.Language attribute) XGA (eval_framework.tasks.base.Language attribute) XGB (eval_framework.tasks.base.Language attribute) XGD (eval_framework.tasks.base.Language attribute) XGF (eval_framework.tasks.base.Language attribute) XGG (eval_framework.tasks.base.Language attribute) XGI (eval_framework.tasks.base.Language attribute) XGL (eval_framework.tasks.base.Language attribute) XGM (eval_framework.tasks.base.Language attribute) XGR (eval_framework.tasks.base.Language attribute) XGU (eval_framework.tasks.base.Language attribute) XGW (eval_framework.tasks.base.Language attribute) XHA (eval_framework.tasks.base.Language attribute) XHC (eval_framework.tasks.base.Language attribute) XHD (eval_framework.tasks.base.Language attribute) XHE (eval_framework.tasks.base.Language attribute) XHM (eval_framework.tasks.base.Language attribute) XHO (eval_framework.tasks.base.Language attribute) XHR (eval_framework.tasks.base.Language attribute) XHT (eval_framework.tasks.base.Language attribute) XHU (eval_framework.tasks.base.Language attribute) XHV (eval_framework.tasks.base.Language attribute) XIA (eval_framework.tasks.base.Language attribute) XIB (eval_framework.tasks.base.Language attribute) XII (eval_framework.tasks.base.Language attribute) XIL (eval_framework.tasks.base.Language attribute) XIN (eval_framework.tasks.base.Language attribute) XIP (eval_framework.tasks.base.Language attribute) XIR (eval_framework.tasks.base.Language attribute) XIS (eval_framework.tasks.base.Language attribute) XIV (eval_framework.tasks.base.Language attribute) XIY (eval_framework.tasks.base.Language attribute) XJB (eval_framework.tasks.base.Language attribute) XJT (eval_framework.tasks.base.Language attribute) XKA (eval_framework.tasks.base.Language attribute) XKB (eval_framework.tasks.base.Language attribute) XKC (eval_framework.tasks.base.Language attribute) XKD (eval_framework.tasks.base.Language attribute) XKE (eval_framework.tasks.base.Language attribute) XKF (eval_framework.tasks.base.Language attribute) XKG (eval_framework.tasks.base.Language attribute) XKH (eval_framework.tasks.base.Language attribute) XKI (eval_framework.tasks.base.Language attribute) XKJ (eval_framework.tasks.base.Language attribute) XKK (eval_framework.tasks.base.Language attribute) XKL (eval_framework.tasks.base.Language attribute) XKM (eval_framework.tasks.base.Language attribute) XKN (eval_framework.tasks.base.Language attribute) XKO (eval_framework.tasks.base.Language attribute) XKP (eval_framework.tasks.base.Language attribute) XKQ (eval_framework.tasks.base.Language attribute) XKR (eval_framework.tasks.base.Language attribute) XKS (eval_framework.tasks.base.Language attribute) XKT (eval_framework.tasks.base.Language attribute) XKU (eval_framework.tasks.base.Language attribute) XKV (eval_framework.tasks.base.Language attribute) XKW (eval_framework.tasks.base.Language attribute) XKX (eval_framework.tasks.base.Language attribute) XKY (eval_framework.tasks.base.Language attribute) XKZ (eval_framework.tasks.base.Language attribute) XLA (eval_framework.tasks.base.Language attribute) XLB (eval_framework.tasks.base.Language attribute) XLC (eval_framework.tasks.base.Language attribute) XLD (eval_framework.tasks.base.Language attribute) XLE (eval_framework.tasks.base.Language attribute) XLG (eval_framework.tasks.base.Language attribute) XLI (eval_framework.tasks.base.Language attribute) XLN (eval_framework.tasks.base.Language attribute) XLO (eval_framework.tasks.base.Language attribute) XLP (eval_framework.tasks.base.Language attribute) XLS (eval_framework.tasks.base.Language attribute) XLU (eval_framework.tasks.base.Language attribute) XLY (eval_framework.tasks.base.Language attribute) XMA (eval_framework.tasks.base.Language attribute) XMB (eval_framework.tasks.base.Language attribute) XMC (eval_framework.tasks.base.Language attribute) XMD (eval_framework.tasks.base.Language attribute) XME (eval_framework.tasks.base.Language attribute) XMF (eval_framework.tasks.base.Language attribute) XMG (eval_framework.tasks.base.Language attribute) XMH (eval_framework.tasks.base.Language attribute) XMI (eval_framework.tasks.base.Language attribute) XMJ (eval_framework.tasks.base.Language attribute) XMK (eval_framework.tasks.base.Language attribute) XML (eval_framework.tasks.base.Language attribute) XMM (eval_framework.tasks.base.Language attribute) XMN (eval_framework.tasks.base.Language attribute) XMO (eval_framework.tasks.base.Language attribute) XMP (eval_framework.tasks.base.Language attribute) XMQ (eval_framework.tasks.base.Language attribute) XMR (eval_framework.tasks.base.Language attribute) XMS (eval_framework.tasks.base.Language attribute) XMT (eval_framework.tasks.base.Language attribute) XMU (eval_framework.tasks.base.Language attribute) XMV (eval_framework.tasks.base.Language attribute) XMW (eval_framework.tasks.base.Language attribute) XMX (eval_framework.tasks.base.Language attribute) XMY (eval_framework.tasks.base.Language attribute) XMZ (eval_framework.tasks.base.Language attribute) XNA (eval_framework.tasks.base.Language attribute) XNB (eval_framework.tasks.base.Language attribute) XNG (eval_framework.tasks.base.Language attribute) XNH (eval_framework.tasks.base.Language attribute) XNI (eval_framework.tasks.base.Language attribute) XNJ (eval_framework.tasks.base.Language attribute) XNK (eval_framework.tasks.base.Language attribute) XNM (eval_framework.tasks.base.Language attribute) XNN (eval_framework.tasks.base.Language attribute) XNO (eval_framework.tasks.base.Language attribute) XNQ (eval_framework.tasks.base.Language attribute) XNR (eval_framework.tasks.base.Language attribute) XNS (eval_framework.tasks.base.Language attribute) XNT (eval_framework.tasks.base.Language attribute) XNU (eval_framework.tasks.base.Language attribute) XNY (eval_framework.tasks.base.Language attribute) XNZ (eval_framework.tasks.base.Language attribute) XOC (eval_framework.tasks.base.Language attribute) XOD (eval_framework.tasks.base.Language attribute) XOG (eval_framework.tasks.base.Language attribute) XOI (eval_framework.tasks.base.Language attribute) XOK (eval_framework.tasks.base.Language attribute) XOM (eval_framework.tasks.base.Language attribute) XON (eval_framework.tasks.base.Language attribute) XOO (eval_framework.tasks.base.Language attribute) XOP (eval_framework.tasks.base.Language attribute) XOR (eval_framework.tasks.base.Language attribute) XOW (eval_framework.tasks.base.Language attribute) XPA (eval_framework.tasks.base.Language attribute) XPB (eval_framework.tasks.base.Language attribute) XPC (eval_framework.tasks.base.Language attribute) XPD (eval_framework.tasks.base.Language attribute) XPE (eval_framework.tasks.base.Language attribute) XPF (eval_framework.tasks.base.Language attribute) XPG (eval_framework.tasks.base.Language attribute) XPH (eval_framework.tasks.base.Language attribute) XPI (eval_framework.tasks.base.Language attribute) XPJ (eval_framework.tasks.base.Language attribute) XPK (eval_framework.tasks.base.Language attribute) XPL (eval_framework.tasks.base.Language attribute) XPM (eval_framework.tasks.base.Language attribute) XPN (eval_framework.tasks.base.Language attribute) XPO (eval_framework.tasks.base.Language attribute) XPP (eval_framework.tasks.base.Language attribute) XPQ (eval_framework.tasks.base.Language attribute) XPR (eval_framework.tasks.base.Language attribute) XPS (eval_framework.tasks.base.Language attribute) XPT (eval_framework.tasks.base.Language attribute) XPU (eval_framework.tasks.base.Language attribute) XPV (eval_framework.tasks.base.Language attribute) XPW (eval_framework.tasks.base.Language attribute) XPX (eval_framework.tasks.base.Language attribute) XPY (eval_framework.tasks.base.Language attribute) XPZ (eval_framework.tasks.base.Language attribute) XQA (eval_framework.tasks.base.Language attribute) XQT (eval_framework.tasks.base.Language attribute) XRA (eval_framework.tasks.base.Language attribute) XRB (eval_framework.tasks.base.Language attribute) XRD (eval_framework.tasks.base.Language attribute) XRE (eval_framework.tasks.base.Language attribute) XRG (eval_framework.tasks.base.Language attribute) XRI (eval_framework.tasks.base.Language attribute) XRM (eval_framework.tasks.base.Language attribute) XRN (eval_framework.tasks.base.Language attribute) XRQ (eval_framework.tasks.base.Language attribute) XRR (eval_framework.tasks.base.Language attribute) XRT (eval_framework.tasks.base.Language attribute) XRU (eval_framework.tasks.base.Language attribute) XRW (eval_framework.tasks.base.Language attribute) XSA (eval_framework.tasks.base.Language attribute) XSB (eval_framework.tasks.base.Language attribute) XSC (eval_framework.tasks.base.Language attribute) XSD (eval_framework.tasks.base.Language attribute) XSE (eval_framework.tasks.base.Language attribute) XSH (eval_framework.tasks.base.Language attribute) XSI (eval_framework.tasks.base.Language attribute) XSJ (eval_framework.tasks.base.Language attribute) XSK (eval_framework.tasks.base.Language attribute) XSL (eval_framework.tasks.base.Language attribute) XSM (eval_framework.tasks.base.Language attribute) XSN (eval_framework.tasks.base.Language attribute) XSO (eval_framework.tasks.base.Language attribute) XSP (eval_framework.tasks.base.Language attribute) XSQ (eval_framework.tasks.base.Language attribute) XSR (eval_framework.tasks.base.Language attribute) XSS (eval_framework.tasks.base.Language attribute) XST (eval_framework.tasks.base.Language attribute) XSU (eval_framework.tasks.base.Language attribute) XSV (eval_framework.tasks.base.Language attribute) XSY (eval_framework.tasks.base.Language attribute) XTA (eval_framework.tasks.base.Language attribute) XTB (eval_framework.tasks.base.Language attribute) XTC (eval_framework.tasks.base.Language attribute) XTD (eval_framework.tasks.base.Language attribute) XTE (eval_framework.tasks.base.Language attribute) XTG (eval_framework.tasks.base.Language attribute) XTH (eval_framework.tasks.base.Language attribute) XTI (eval_framework.tasks.base.Language attribute) XTJ (eval_framework.tasks.base.Language attribute) XTL (eval_framework.tasks.base.Language attribute) XTM (eval_framework.tasks.base.Language attribute) XTN (eval_framework.tasks.base.Language attribute) XTO (eval_framework.tasks.base.Language attribute) XTP (eval_framework.tasks.base.Language attribute) XTQ (eval_framework.tasks.base.Language attribute) XTR (eval_framework.tasks.base.Language attribute) XTS (eval_framework.tasks.base.Language attribute) XTT (eval_framework.tasks.base.Language attribute) XTU (eval_framework.tasks.base.Language attribute) XTV (eval_framework.tasks.base.Language attribute) XTW (eval_framework.tasks.base.Language attribute) XTY (eval_framework.tasks.base.Language attribute) XTZ (eval_framework.tasks.base.Language attribute) XUA (eval_framework.tasks.base.Language attribute) XUB (eval_framework.tasks.base.Language attribute) XUD (eval_framework.tasks.base.Language attribute) XUF (eval_framework.tasks.base.Language attribute) XUG (eval_framework.tasks.base.Language attribute) XUJ (eval_framework.tasks.base.Language attribute) XUL (eval_framework.tasks.base.Language attribute) XUM (eval_framework.tasks.base.Language attribute) XUN (eval_framework.tasks.base.Language attribute) XUO (eval_framework.tasks.base.Language attribute) XUP (eval_framework.tasks.base.Language attribute) XUR (eval_framework.tasks.base.Language attribute) XUT (eval_framework.tasks.base.Language attribute) XUU (eval_framework.tasks.base.Language attribute) XVE (eval_framework.tasks.base.Language attribute) XVI (eval_framework.tasks.base.Language attribute) XVN (eval_framework.tasks.base.Language attribute) XVO (eval_framework.tasks.base.Language attribute) XVS (eval_framework.tasks.base.Language attribute) XWA (eval_framework.tasks.base.Language attribute) XWC (eval_framework.tasks.base.Language attribute) XWD (eval_framework.tasks.base.Language attribute) XWE (eval_framework.tasks.base.Language attribute) XWG (eval_framework.tasks.base.Language attribute) XWJ (eval_framework.tasks.base.Language attribute) XWK (eval_framework.tasks.base.Language attribute) XWL (eval_framework.tasks.base.Language attribute) XWO (eval_framework.tasks.base.Language attribute) XWR (eval_framework.tasks.base.Language attribute) XWT (eval_framework.tasks.base.Language attribute) XWW (eval_framework.tasks.base.Language attribute) XXB (eval_framework.tasks.base.Language attribute) XXK (eval_framework.tasks.base.Language attribute) XXM (eval_framework.tasks.base.Language attribute) XXR (eval_framework.tasks.base.Language attribute) XXT (eval_framework.tasks.base.Language attribute) XYA (eval_framework.tasks.base.Language attribute) XYB (eval_framework.tasks.base.Language attribute) XYJ (eval_framework.tasks.base.Language attribute) XYK (eval_framework.tasks.base.Language attribute) XYL (eval_framework.tasks.base.Language attribute) XYT (eval_framework.tasks.base.Language attribute) XYY (eval_framework.tasks.base.Language attribute) XZH (eval_framework.tasks.base.Language attribute) XZM (eval_framework.tasks.base.Language attribute) XZP (eval_framework.tasks.base.Language attribute) Y YAA (eval_framework.tasks.base.Language attribute) YAB (eval_framework.tasks.base.Language attribute) YAC (eval_framework.tasks.base.Language attribute) YAD (eval_framework.tasks.base.Language attribute) YAE (eval_framework.tasks.base.Language attribute) YAF (eval_framework.tasks.base.Language attribute) YAG (eval_framework.tasks.base.Language attribute) YAH (eval_framework.tasks.base.Language attribute) YAI (eval_framework.tasks.base.Language attribute) YAJ (eval_framework.tasks.base.Language attribute) YAK (eval_framework.tasks.base.Language attribute) YAL (eval_framework.tasks.base.Language attribute) YAM (eval_framework.tasks.base.Language attribute) YAN (eval_framework.tasks.base.Language attribute) YAO (eval_framework.tasks.base.Language attribute) YAP (eval_framework.tasks.base.Language attribute) YAQ (eval_framework.tasks.base.Language attribute) YAR (eval_framework.tasks.base.Language attribute) YAS (eval_framework.tasks.base.Language attribute) YAT (eval_framework.tasks.base.Language attribute) YAU (eval_framework.tasks.base.Language attribute) YAV (eval_framework.tasks.base.Language attribute) YAW (eval_framework.tasks.base.Language attribute) YAX (eval_framework.tasks.base.Language attribute) YAY (eval_framework.tasks.base.Language attribute) YAZ (eval_framework.tasks.base.Language attribute) YBA (eval_framework.tasks.base.Language attribute) YBB (eval_framework.tasks.base.Language attribute) YBD (eval_framework.tasks.base.Language attribute) YBE (eval_framework.tasks.base.Language attribute) YBH (eval_framework.tasks.base.Language attribute) YBI (eval_framework.tasks.base.Language attribute) YBJ (eval_framework.tasks.base.Language attribute) YBK (eval_framework.tasks.base.Language attribute) YBL (eval_framework.tasks.base.Language attribute) YBM (eval_framework.tasks.base.Language attribute) YBN (eval_framework.tasks.base.Language attribute) YBO (eval_framework.tasks.base.Language attribute) YBX (eval_framework.tasks.base.Language attribute) YBY (eval_framework.tasks.base.Language attribute) YCH (eval_framework.tasks.base.Language attribute) YCL (eval_framework.tasks.base.Language attribute) YCN (eval_framework.tasks.base.Language attribute) YCP (eval_framework.tasks.base.Language attribute) YCR (eval_framework.tasks.base.Language attribute) YDA (eval_framework.tasks.base.Language attribute) YDD (eval_framework.tasks.base.Language attribute) YDE (eval_framework.tasks.base.Language attribute) YDG (eval_framework.tasks.base.Language attribute) YDK (eval_framework.tasks.base.Language attribute) YDS (eval_framework.tasks.base.Language attribute) YEA (eval_framework.tasks.base.Language attribute) YEC (eval_framework.tasks.base.Language attribute) YEE (eval_framework.tasks.base.Language attribute) YEI (eval_framework.tasks.base.Language attribute) YEJ (eval_framework.tasks.base.Language attribute) YEL (eval_framework.tasks.base.Language attribute) YEN (eval_framework.tasks.base.Language attribute) YER (eval_framework.tasks.base.Language attribute) YES (eval_framework.tasks.base.Language attribute) YET (eval_framework.tasks.base.Language attribute) YEU (eval_framework.tasks.base.Language attribute) YEV (eval_framework.tasks.base.Language attribute) YEY (eval_framework.tasks.base.Language attribute) YGA (eval_framework.tasks.base.Language attribute) YGI (eval_framework.tasks.base.Language attribute) YGL (eval_framework.tasks.base.Language attribute) YGM (eval_framework.tasks.base.Language attribute) YGP (eval_framework.tasks.base.Language attribute) YGR (eval_framework.tasks.base.Language attribute) YGS (eval_framework.tasks.base.Language attribute) YGU (eval_framework.tasks.base.Language attribute) YGW (eval_framework.tasks.base.Language attribute) YHA (eval_framework.tasks.base.Language attribute) YHD (eval_framework.tasks.base.Language attribute) YHL (eval_framework.tasks.base.Language attribute) YHS (eval_framework.tasks.base.Language attribute) YIA (eval_framework.tasks.base.Language attribute) YIB (eval_framework.tasks.base.Language attribute) YID (eval_framework.tasks.base.Language attribute) YIF (eval_framework.tasks.base.Language attribute) YIG (eval_framework.tasks.base.Language attribute) YIH (eval_framework.tasks.base.Language attribute) YII (eval_framework.tasks.base.Language attribute) YIJ (eval_framework.tasks.base.Language attribute) YIK (eval_framework.tasks.base.Language attribute) YIL (eval_framework.tasks.base.Language attribute) YIM (eval_framework.tasks.base.Language attribute) YIN (eval_framework.tasks.base.Language attribute) YIO (eval_framework.tasks.base.Language attribute) YIP (eval_framework.tasks.base.Language attribute) YIQ (eval_framework.tasks.base.Language attribute) YIR (eval_framework.tasks.base.Language attribute) YIS (eval_framework.tasks.base.Language attribute) YIT (eval_framework.tasks.base.Language attribute) YIU (eval_framework.tasks.base.Language attribute) YIV (eval_framework.tasks.base.Language attribute) YIX (eval_framework.tasks.base.Language attribute) YIY (eval_framework.tasks.base.Language attribute) YIZ (eval_framework.tasks.base.Language attribute) YKA (eval_framework.tasks.base.Language attribute) YKG (eval_framework.tasks.base.Language attribute) YKH (eval_framework.tasks.base.Language attribute) YKI (eval_framework.tasks.base.Language attribute) YKK (eval_framework.tasks.base.Language attribute) YKL (eval_framework.tasks.base.Language attribute) YKM (eval_framework.tasks.base.Language attribute) YKN (eval_framework.tasks.base.Language attribute) YKO (eval_framework.tasks.base.Language attribute) YKR (eval_framework.tasks.base.Language attribute) YKT (eval_framework.tasks.base.Language attribute) YKU (eval_framework.tasks.base.Language attribute) YKY (eval_framework.tasks.base.Language attribute) YLA (eval_framework.tasks.base.Language attribute) YLB (eval_framework.tasks.base.Language attribute) YLE (eval_framework.tasks.base.Language attribute) YLG (eval_framework.tasks.base.Language attribute) YLI (eval_framework.tasks.base.Language attribute) YLL (eval_framework.tasks.base.Language attribute) YLM (eval_framework.tasks.base.Language attribute) YLN (eval_framework.tasks.base.Language attribute) YLO (eval_framework.tasks.base.Language attribute) YLR (eval_framework.tasks.base.Language attribute) YLU (eval_framework.tasks.base.Language attribute) YLY (eval_framework.tasks.base.Language attribute) YMA (eval_framework.tasks.base.Language attribute) YMB (eval_framework.tasks.base.Language attribute) YMC (eval_framework.tasks.base.Language attribute) YMD (eval_framework.tasks.base.Language attribute) YME (eval_framework.tasks.base.Language attribute) YMG (eval_framework.tasks.base.Language attribute) YMH (eval_framework.tasks.base.Language attribute) YMI (eval_framework.tasks.base.Language attribute) YMJ (eval_framework.tasks.base.Language attribute) YMK (eval_framework.tasks.base.Language attribute) YML (eval_framework.tasks.base.Language attribute) YMM (eval_framework.tasks.base.Language attribute) YMN (eval_framework.tasks.base.Language attribute) YMO (eval_framework.tasks.base.Language attribute) YMP (eval_framework.tasks.base.Language attribute) YMQ (eval_framework.tasks.base.Language attribute) YMR (eval_framework.tasks.base.Language attribute) YMS (eval_framework.tasks.base.Language attribute) YMT (eval_framework.tasks.base.Language attribute) YMX (eval_framework.tasks.base.Language attribute) YMZ (eval_framework.tasks.base.Language attribute) YNA (eval_framework.tasks.base.Language attribute) YNB (eval_framework.tasks.base.Language attribute) YND (eval_framework.tasks.base.Language attribute) YNE (eval_framework.tasks.base.Language attribute) YNG (eval_framework.tasks.base.Language attribute) YNH (eval_framework.tasks.base.Language attribute) YNK (eval_framework.tasks.base.Language attribute) YNL (eval_framework.tasks.base.Language attribute) YNN (eval_framework.tasks.base.Language attribute) YNO (eval_framework.tasks.base.Language attribute) YNQ (eval_framework.tasks.base.Language attribute) YNS (eval_framework.tasks.base.Language attribute) YNU (eval_framework.tasks.base.Language attribute) YOB (eval_framework.tasks.base.Language attribute) YOG (eval_framework.tasks.base.Language attribute) YOI (eval_framework.tasks.base.Language attribute) YOK (eval_framework.tasks.base.Language attribute) YOL (eval_framework.tasks.base.Language attribute) YOM (eval_framework.tasks.base.Language attribute) YON (eval_framework.tasks.base.Language attribute) YOR (eval_framework.tasks.base.Language attribute) YOS (eval_framework.tasks.base.Language attribute) YOT (eval_framework.tasks.base.Language attribute) YOX (eval_framework.tasks.base.Language attribute) YOY (eval_framework.tasks.base.Language attribute) YPA (eval_framework.tasks.base.Language attribute) YPB (eval_framework.tasks.base.Language attribute) YPG (eval_framework.tasks.base.Language attribute) YPH (eval_framework.tasks.base.Language attribute) YPL (eval_framework.tasks.base.Language attribute) YPM (eval_framework.tasks.base.Language attribute) YPN (eval_framework.tasks.base.Language attribute) YPO (eval_framework.tasks.base.Language attribute) YPP (eval_framework.tasks.base.Language attribute) YPW (eval_framework.tasks.base.Language attribute) YPZ (eval_framework.tasks.base.Language attribute) YRA (eval_framework.tasks.base.Language attribute) YRB (eval_framework.tasks.base.Language attribute) YRE (eval_framework.tasks.base.Language attribute) YRI (eval_framework.tasks.base.Language attribute) YRK (eval_framework.tasks.base.Language attribute) YRL (eval_framework.tasks.base.Language attribute) YRM (eval_framework.tasks.base.Language attribute) YRN (eval_framework.tasks.base.Language attribute) YRO (eval_framework.tasks.base.Language attribute) YRS (eval_framework.tasks.base.Language attribute) YRW (eval_framework.tasks.base.Language attribute) YRY (eval_framework.tasks.base.Language attribute) YSC (eval_framework.tasks.base.Language attribute) YSD (eval_framework.tasks.base.Language attribute) YSG (eval_framework.tasks.base.Language attribute) YSL (eval_framework.tasks.base.Language attribute) YSM (eval_framework.tasks.base.Language attribute) YSN (eval_framework.tasks.base.Language attribute) YSO (eval_framework.tasks.base.Language attribute) YSP (eval_framework.tasks.base.Language attribute) YSR (eval_framework.tasks.base.Language attribute) YSS (eval_framework.tasks.base.Language attribute) YSY (eval_framework.tasks.base.Language attribute) YTA (eval_framework.tasks.base.Language attribute) YTL (eval_framework.tasks.base.Language attribute) YTP (eval_framework.tasks.base.Language attribute) YTW (eval_framework.tasks.base.Language attribute) YTY (eval_framework.tasks.base.Language attribute) YUA (eval_framework.tasks.base.Language attribute) YUB (eval_framework.tasks.base.Language attribute) YUC (eval_framework.tasks.base.Language attribute) YUD (eval_framework.tasks.base.Language attribute) YUE (eval_framework.tasks.base.Language attribute) YUF (eval_framework.tasks.base.Language attribute) YUG (eval_framework.tasks.base.Language attribute) YUI (eval_framework.tasks.base.Language attribute) YUJ (eval_framework.tasks.base.Language attribute) YUK (eval_framework.tasks.base.Language attribute) YUL (eval_framework.tasks.base.Language attribute) YUM (eval_framework.tasks.base.Language attribute) YUN (eval_framework.tasks.base.Language attribute) YUP (eval_framework.tasks.base.Language attribute) YUQ (eval_framework.tasks.base.Language attribute) YUR (eval_framework.tasks.base.Language attribute) YUS (eval_framework.tasks.base.Language attribute) YUT (eval_framework.tasks.base.Language attribute) YUU (eval_framework.tasks.base.Language attribute) YUW (eval_framework.tasks.base.Language attribute) YUX (eval_framework.tasks.base.Language attribute) YUY (eval_framework.tasks.base.Language attribute) YUZ (eval_framework.tasks.base.Language attribute) YVA (eval_framework.tasks.base.Language attribute) YVT (eval_framework.tasks.base.Language attribute) YWA (eval_framework.tasks.base.Language attribute) YWG (eval_framework.tasks.base.Language attribute) YWL (eval_framework.tasks.base.Language attribute) YWM (eval_framework.tasks.base.Language attribute) YWN (eval_framework.tasks.base.Language attribute) YWQ (eval_framework.tasks.base.Language attribute) YWR (eval_framework.tasks.base.Language attribute) YWT (eval_framework.tasks.base.Language attribute) YWU (eval_framework.tasks.base.Language attribute) YWW (eval_framework.tasks.base.Language attribute) YXA (eval_framework.tasks.base.Language attribute) YXG (eval_framework.tasks.base.Language attribute) YXL (eval_framework.tasks.base.Language attribute) YXM (eval_framework.tasks.base.Language attribute) YXU (eval_framework.tasks.base.Language attribute) YXY (eval_framework.tasks.base.Language attribute) YYM (eval_framework.tasks.base.Language attribute) YYR (eval_framework.tasks.base.Language attribute) YYU (eval_framework.tasks.base.Language attribute) YYZ (eval_framework.tasks.base.Language attribute) YZG (eval_framework.tasks.base.Language attribute) YZK (eval_framework.tasks.base.Language attribute) Z ZAA (eval_framework.tasks.base.Language attribute) ZAB (eval_framework.tasks.base.Language attribute) ZAC (eval_framework.tasks.base.Language attribute) ZAD (eval_framework.tasks.base.Language attribute) ZAE (eval_framework.tasks.base.Language attribute) ZAF (eval_framework.tasks.base.Language attribute) ZAG (eval_framework.tasks.base.Language attribute) ZAH (eval_framework.tasks.base.Language attribute) ZAI (eval_framework.tasks.base.Language attribute) ZAJ (eval_framework.tasks.base.Language attribute) ZAK (eval_framework.tasks.base.Language attribute) ZAL (eval_framework.tasks.base.Language attribute) ZAM (eval_framework.tasks.base.Language attribute) ZAO (eval_framework.tasks.base.Language attribute) ZAP (eval_framework.tasks.base.Language attribute) ZAQ (eval_framework.tasks.base.Language attribute) ZAR (eval_framework.tasks.base.Language attribute) ZAS (eval_framework.tasks.base.Language attribute) ZAT (eval_framework.tasks.base.Language attribute) ZAU (eval_framework.tasks.base.Language attribute) ZAV (eval_framework.tasks.base.Language attribute) ZAW (eval_framework.tasks.base.Language attribute) ZAX (eval_framework.tasks.base.Language attribute) ZAY (eval_framework.tasks.base.Language attribute) ZAZ (eval_framework.tasks.base.Language attribute) ZBA (eval_framework.tasks.base.Language attribute) ZBC (eval_framework.tasks.base.Language attribute) ZBE (eval_framework.tasks.base.Language attribute) ZBL (eval_framework.tasks.base.Language attribute) ZBT (eval_framework.tasks.base.Language attribute) ZBU (eval_framework.tasks.base.Language attribute) ZBW (eval_framework.tasks.base.Language attribute) ZCA (eval_framework.tasks.base.Language attribute) ZCD (eval_framework.tasks.base.Language attribute) ZCH (eval_framework.tasks.base.Language attribute) ZDJ (eval_framework.tasks.base.Language attribute) ZEA (eval_framework.tasks.base.Language attribute) ZEG (eval_framework.tasks.base.Language attribute) ZEH (eval_framework.tasks.base.Language attribute) ZEM (eval_framework.tasks.base.Language attribute) ZEN (eval_framework.tasks.base.Language attribute) ZERO_SCROLLS_COMPLETION (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_GOV_REPORT (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_MUSIQUE (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_NARRATIVEQA (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_QASPER (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_QMSUM (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_QUALITY (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_SPACE_DIGEST (class in eval_framework.tasks.benchmarks.zero_scrolls) ZERO_SCROLLS_SQUALITY (class in eval_framework.tasks.benchmarks.zero_scrolls) ZGA (eval_framework.tasks.base.Language attribute) ZGB (eval_framework.tasks.base.Language attribute) ZGH (eval_framework.tasks.base.Language attribute) ZGM (eval_framework.tasks.base.Language attribute) ZGN (eval_framework.tasks.base.Language attribute) ZGR (eval_framework.tasks.base.Language attribute) ZHA (eval_framework.tasks.base.Language attribute) ZHB (eval_framework.tasks.base.Language attribute) ZHD (eval_framework.tasks.base.Language attribute) ZHI (eval_framework.tasks.base.Language attribute) ZHN (eval_framework.tasks.base.Language attribute) ZHO (eval_framework.tasks.base.Language attribute) ZHW (eval_framework.tasks.base.Language attribute) ZIA (eval_framework.tasks.base.Language attribute) ZIB (eval_framework.tasks.base.Language attribute) ZIK (eval_framework.tasks.base.Language attribute) ZIL (eval_framework.tasks.base.Language attribute) ZIM (eval_framework.tasks.base.Language attribute) ZIN (eval_framework.tasks.base.Language attribute) ZIR (eval_framework.tasks.base.Language attribute) ZIW (eval_framework.tasks.base.Language attribute) ZIZ (eval_framework.tasks.base.Language attribute) ZKA (eval_framework.tasks.base.Language attribute) ZKB (eval_framework.tasks.base.Language attribute) ZKD (eval_framework.tasks.base.Language attribute) ZKG (eval_framework.tasks.base.Language attribute) ZKH (eval_framework.tasks.base.Language attribute) ZKK (eval_framework.tasks.base.Language attribute) ZKN (eval_framework.tasks.base.Language attribute) ZKO (eval_framework.tasks.base.Language attribute) ZKP (eval_framework.tasks.base.Language attribute) ZKR (eval_framework.tasks.base.Language attribute) ZKT (eval_framework.tasks.base.Language attribute) ZKU (eval_framework.tasks.base.Language attribute) ZKV (eval_framework.tasks.base.Language attribute) ZKZ (eval_framework.tasks.base.Language attribute) ZLA (eval_framework.tasks.base.Language attribute) ZLJ (eval_framework.tasks.base.Language attribute) ZLM (eval_framework.tasks.base.Language attribute) ZLN (eval_framework.tasks.base.Language attribute) ZLQ (eval_framework.tasks.base.Language attribute) ZLU (eval_framework.tasks.base.Language attribute) ZMA (eval_framework.tasks.base.Language attribute) ZMB (eval_framework.tasks.base.Language attribute) ZMC (eval_framework.tasks.base.Language attribute) ZMD (eval_framework.tasks.base.Language attribute) ZME (eval_framework.tasks.base.Language attribute) ZMF (eval_framework.tasks.base.Language attribute) ZMG (eval_framework.tasks.base.Language attribute) ZMH (eval_framework.tasks.base.Language attribute) ZMI (eval_framework.tasks.base.Language attribute) ZMJ (eval_framework.tasks.base.Language attribute) ZMK (eval_framework.tasks.base.Language attribute) ZML (eval_framework.tasks.base.Language attribute) ZMM (eval_framework.tasks.base.Language attribute) ZMN (eval_framework.tasks.base.Language attribute) ZMO (eval_framework.tasks.base.Language attribute) ZMP (eval_framework.tasks.base.Language attribute) ZMQ (eval_framework.tasks.base.Language attribute) ZMR (eval_framework.tasks.base.Language attribute) ZMS (eval_framework.tasks.base.Language attribute) ZMT (eval_framework.tasks.base.Language attribute) ZMU (eval_framework.tasks.base.Language attribute) ZMV (eval_framework.tasks.base.Language attribute) ZMW (eval_framework.tasks.base.Language attribute) ZMX (eval_framework.tasks.base.Language attribute) ZMY (eval_framework.tasks.base.Language attribute) ZMZ (eval_framework.tasks.base.Language attribute) ZNA (eval_framework.tasks.base.Language attribute) ZNE (eval_framework.tasks.base.Language attribute) ZNG (eval_framework.tasks.base.Language attribute) ZNK (eval_framework.tasks.base.Language attribute) ZNS (eval_framework.tasks.base.Language attribute) ZOC (eval_framework.tasks.base.Language attribute) ZOH (eval_framework.tasks.base.Language attribute) ZOM (eval_framework.tasks.base.Language attribute) ZOO (eval_framework.tasks.base.Language attribute) ZOQ (eval_framework.tasks.base.Language attribute) ZOR (eval_framework.tasks.base.Language attribute) ZOS (eval_framework.tasks.base.Language attribute) ZPA (eval_framework.tasks.base.Language attribute) ZPB (eval_framework.tasks.base.Language attribute) ZPC (eval_framework.tasks.base.Language attribute) ZPD (eval_framework.tasks.base.Language attribute) ZPE (eval_framework.tasks.base.Language attribute) ZPF (eval_framework.tasks.base.Language attribute) ZPG (eval_framework.tasks.base.Language attribute) ZPH (eval_framework.tasks.base.Language attribute) ZPI (eval_framework.tasks.base.Language attribute) ZPJ (eval_framework.tasks.base.Language attribute) ZPK (eval_framework.tasks.base.Language attribute) ZPL (eval_framework.tasks.base.Language attribute) ZPM (eval_framework.tasks.base.Language attribute) ZPN (eval_framework.tasks.base.Language attribute) ZPO (eval_framework.tasks.base.Language attribute) ZPP (eval_framework.tasks.base.Language attribute) ZPQ (eval_framework.tasks.base.Language attribute) ZPR (eval_framework.tasks.base.Language attribute) ZPS (eval_framework.tasks.base.Language attribute) ZPT (eval_framework.tasks.base.Language attribute) ZPU (eval_framework.tasks.base.Language attribute) ZPV (eval_framework.tasks.base.Language attribute) ZPW (eval_framework.tasks.base.Language attribute) ZPX (eval_framework.tasks.base.Language attribute) ZPY (eval_framework.tasks.base.Language attribute) ZPZ (eval_framework.tasks.base.Language attribute) ZQE (eval_framework.tasks.base.Language attribute) ZRA (eval_framework.tasks.base.Language attribute) ZRG (eval_framework.tasks.base.Language attribute) ZRN (eval_framework.tasks.base.Language attribute) ZRO (eval_framework.tasks.base.Language attribute) ZRP (eval_framework.tasks.base.Language attribute) ZRS (eval_framework.tasks.base.Language attribute) ZSA (eval_framework.tasks.base.Language attribute) ZSK (eval_framework.tasks.base.Language attribute) ZSL (eval_framework.tasks.base.Language attribute) ZSM (eval_framework.tasks.base.Language attribute) ZSR (eval_framework.tasks.base.Language attribute) ZSU (eval_framework.tasks.base.Language attribute) ZTC (eval_framework.tasks.base.Language attribute) ZTE (eval_framework.tasks.base.Language attribute) ZTG (eval_framework.tasks.base.Language attribute) ZTL (eval_framework.tasks.base.Language attribute) ZTM (eval_framework.tasks.base.Language attribute) ZTN (eval_framework.tasks.base.Language attribute) ZTP (eval_framework.tasks.base.Language attribute) ZTQ (eval_framework.tasks.base.Language attribute) ZTS (eval_framework.tasks.base.Language attribute) ZTT (eval_framework.tasks.base.Language attribute) ZTU (eval_framework.tasks.base.Language attribute) ZTX (eval_framework.tasks.base.Language attribute) ZTY (eval_framework.tasks.base.Language attribute) ZUA (eval_framework.tasks.base.Language attribute) ZUH (eval_framework.tasks.base.Language attribute) ZUL (eval_framework.tasks.base.Language attribute) ZUM (eval_framework.tasks.base.Language attribute) ZUN (eval_framework.tasks.base.Language attribute) ZUY (eval_framework.tasks.base.Language attribute) ZWA (eval_framework.tasks.base.Language attribute) ZXX (eval_framework.tasks.base.Language attribute) ZYB (eval_framework.tasks.base.Language attribute) ZYG (eval_framework.tasks.base.Language attribute) ZYJ (eval_framework.tasks.base.Language attribute) ZYN (eval_framework.tasks.base.Language attribute) ZYP (eval_framework.tasks.base.Language attribute) ZZA (eval_framework.tasks.base.Language attribute) ZZJ (eval_framework.tasks.base.Language attribute)
