GlobalMMLUΒΆ

NAME = GlobalMMLU
DATASET_PATH = CohereLabs/Global-MMLU
SAMPLE_SPLIT = test
FEWSHOT_SPLIT = dev
RESPONSE_TYPE = LOGLIKELIHOODS
METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
SUBJECTS = [('fr', 'abstract_algebra'), ('fr', 'anatomy'), ('fr', 'astronomy'), ('fr', 'business_ethics'), ('fr', 'clinical_knowledge'), ('fr', 'college_biology'), ('fr', 'college_chemistry'), ('fr', 'college_computer_science'), ('fr', 'college_mathematics'), ('fr', 'college_medicine'), ('fr', 'college_physics'), ('fr', 'computer_security'), ('fr', 'conceptual_physics'), ('fr', 'econometrics'), ('fr', 'electrical_engineering'), ('fr', 'elementary_mathematics'), ('fr', 'formal_logic'), ('fr', 'global_facts'), ('fr', 'high_school_biology'), ('fr', 'high_school_chemistry'), ('fr', 'high_school_computer_science'), ('fr', 'high_school_european_history'), ('fr', 'high_school_geography'), ('fr', 'high_school_government_and_politics'), ('fr', 'high_school_macroeconomics'), ('fr', 'high_school_mathematics'), ('fr', 'high_school_microeconomics'), ('fr', 'high_school_physics'), ('fr', 'high_school_psychology'), ('fr', 'high_school_statistics'), ('fr', 'high_school_us_history'), ('fr', 'high_school_world_history'), ('fr', 'human_aging'), ('fr', 'human_sexuality'), ('fr', 'international_law'), ('fr', 'jurisprudence'), ('fr', 'logical_fallacies'), ('fr', 'machine_learning'), ('fr', 'management'), ('fr', 'marketing'), ('fr', 'medical_genetics'), ('fr', 'miscellaneous'), ('fr', 'moral_disputes'), ('fr', 'moral_scenarios'), ('fr', 'nutrition'), ('fr', 'philosophy'), ('fr', 'prehistory'), ('fr', 'professional_accounting'), ('fr', 'professional_law'), ('fr', 'professional_medicine'), ('fr', 'professional_psychology'), ('fr', 'public_relations'), ('fr', 'security_studies'), ('fr', 'sociology'), ('fr', 'us_foreign_policy'), ('fr', 'virology'), ('fr', 'world_religions'), ('de', 'abstract_algebra'), ('de', 'anatomy'), ('de', 'astronomy'), ('de', 'business_ethics'), ('de', 'clinical_knowledge'), ('de', 'college_biology'), ('de', 'college_chemistry'), ('de', 'college_computer_science'), ('de', 'college_mathematics'), ('de', 'college_medicine'), ('de', 'college_physics'), ('de', 'computer_security'), ('de', 'conceptual_physics'), ('de', 'econometrics'), ('de', 'electrical_engineering'), ('de', 'elementary_mathematics'), ('de', 'formal_logic'), ('de', 'global_facts'), ('de', 'high_school_biology'), ('de', 'high_school_chemistry'), ('de', 'high_school_computer_science'), ('de', 'high_school_european_history'), ('de', 'high_school_geography'), ('de', 'high_school_government_and_politics'), ('de', 'high_school_macroeconomics'), ('de', 'high_school_mathematics'), ('de', 'high_school_microeconomics'), ('de', 'high_school_physics'), ('de', 'high_school_psychology'), ('de', 'high_school_statistics'), ('de', 'high_school_us_history'), ('de', 'high_school_world_history'), ('de', 'human_aging'), ('de', 'human_sexuality'), ('de', 'international_law'), ('de', 'jurisprudence'), ('de', 'logical_fallacies'), ('de', 'machine_learning'), ('de', 'management'), ('de', 'marketing'), ('de', 'medical_genetics'), ('de', 'miscellaneous'), ('de', 'moral_disputes'), ('de', 'moral_scenarios'), ('de', 'nutrition'), ('de', 'philosophy'), ('de', 'prehistory'), ('de', 'professional_accounting'), ('de', 'professional_law'), ('de', 'professional_medicine'), ('de', 'professional_psychology'), ('de', 'public_relations'), ('de', 'security_studies'), ('de', 'sociology'), ('de', 'us_foreign_policy'), ('de', 'virology'), ('de', 'world_religions'), ('es', 'abstract_algebra'), ('es', 'anatomy'), ('es', 'astronomy'), ('es', 'business_ethics'), ('es', 'clinical_knowledge'), ('es', 'college_biology'), ('es', 'college_chemistry'), ('es', 'college_computer_science'), ('es', 'college_mathematics'), ('es', 'college_medicine'), ('es', 'college_physics'), ('es', 'computer_security'), ('es', 'conceptual_physics'), ('es', 'econometrics'), ('es', 'electrical_engineering'), ('es', 'elementary_mathematics'), ('es', 'formal_logic'), ('es', 'global_facts'), ('es', 'high_school_biology'), ('es', 'high_school_chemistry'), ('es', 'high_school_computer_science'), ('es', 'high_school_european_history'), ('es', 'high_school_geography'), ('es', 'high_school_government_and_politics'), ('es', 'high_school_macroeconomics'), ('es', 'high_school_mathematics'), ('es', 'high_school_microeconomics'), ('es', 'high_school_physics'), ('es', 'high_school_psychology'), ('es', 'high_school_statistics'), ('es', 'high_school_us_history'), ('es', 'high_school_world_history'), ('es', 'human_aging'), ('es', 'human_sexuality'), ('es', 'international_law'), ('es', 'jurisprudence'), ('es', 'logical_fallacies'), ('es', 'machine_learning'), ('es', 'management'), ('es', 'marketing'), ('es', 'medical_genetics'), ('es', 'miscellaneous'), ('es', 'moral_disputes'), ('es', 'moral_scenarios'), ('es', 'nutrition'), ('es', 'philosophy'), ('es', 'prehistory'), ('es', 'professional_accounting'), ('es', 'professional_law'), ('es', 'professional_medicine'), ('es', 'professional_psychology'), ('es', 'public_relations'), ('es', 'security_studies'), ('es', 'sociology'), ('es', 'us_foreign_policy'), ('es', 'virology'), ('es', 'world_religions'), ('it', 'abstract_algebra'), ('it', 'anatomy'), ('it', 'astronomy'), ('it', 'business_ethics'), ('it', 'clinical_knowledge'), ('it', 'college_biology'), ('it', 'college_chemistry'), ('it', 'college_computer_science'), ('it', 'college_mathematics'), ('it', 'college_medicine'), ('it', 'college_physics'), ('it', 'computer_security'), ('it', 'conceptual_physics'), ('it', 'econometrics'), ('it', 'electrical_engineering'), ('it', 'elementary_mathematics'), ('it', 'formal_logic'), ('it', 'global_facts'), ('it', 'high_school_biology'), ('it', 'high_school_chemistry'), ('it', 'high_school_computer_science'), ('it', 'high_school_european_history'), ('it', 'high_school_geography'), ('it', 'high_school_government_and_politics'), ('it', 'high_school_macroeconomics'), ('it', 'high_school_mathematics'), ('it', 'high_school_microeconomics'), ('it', 'high_school_physics'), ('it', 'high_school_psychology'), ('it', 'high_school_statistics'), ('it', 'high_school_us_history'), ('it', 'high_school_world_history'), ('it', 'human_aging'), ('it', 'human_sexuality'), ('it', 'international_law'), ('it', 'jurisprudence'), ('it', 'logical_fallacies'), ('it', 'machine_learning'), ('it', 'management'), ('it', 'marketing'), ('it', 'medical_genetics'), ('it', 'miscellaneous'), ('it', 'moral_disputes'), ('it', 'moral_scenarios'), ('it', 'nutrition'), ('it', 'philosophy'), ('it', 'prehistory'), ('it', 'professional_accounting'), ('it', 'professional_law'), ('it', 'professional_medicine'), ('it', 'professional_psychology'), ('it', 'public_relations'), ('it', 'security_studies'), ('it', 'sociology'), ('it', 'us_foreign_policy'), ('it', 'virology'), ('it', 'world_religions'), ('pt', 'abstract_algebra'), ('pt', 'anatomy'), ('pt', 'astronomy'), ('pt', 'business_ethics'), ('pt', 'clinical_knowledge'), ('pt', 'college_biology'), ('pt', 'college_chemistry'), ('pt', 'college_computer_science'), ('pt', 'college_mathematics'), ('pt', 'college_medicine'), ('pt', 'college_physics'), ('pt', 'computer_security'), ('pt', 'conceptual_physics'), ('pt', 'econometrics'), ('pt', 'electrical_engineering'), ('pt', 'elementary_mathematics'), ('pt', 'formal_logic'), ('pt', 'global_facts'), ('pt', 'high_school_biology'), ('pt', 'high_school_chemistry'), ('pt', 'high_school_computer_science'), ('pt', 'high_school_european_history'), ('pt', 'high_school_geography'), ('pt', 'high_school_government_and_politics'), ('pt', 'high_school_macroeconomics'), ('pt', 'high_school_mathematics'), ('pt', 'high_school_microeconomics'), ('pt', 'high_school_physics'), ('pt', 'high_school_psychology'), ('pt', 'high_school_statistics'), ('pt', 'high_school_us_history'), ('pt', 'high_school_world_history'), ('pt', 'human_aging'), ('pt', 'human_sexuality'), ('pt', 'international_law'), ('pt', 'jurisprudence'), ('pt', 'logical_fallacies'), ('pt', 'machine_learning'), ('pt', 'management'), ('pt', 'marketing'), ('pt', 'medical_genetics'), ('pt', 'miscellaneous'), ('pt', 'moral_disputes'), ('pt', 'moral_scenarios'), ('pt', 'nutrition'), ('pt', 'philosophy'), ('pt', 'prehistory'), ('pt', 'professional_accounting'), ('pt', 'professional_law'), ('pt', 'professional_medicine'), ('pt', 'professional_psychology'), ('pt', 'public_relations'), ('pt', 'security_studies'), ('pt', 'sociology'), ('pt', 'us_foreign_policy'), ('pt', 'virology'), ('pt', 'world_religions'), ('ar', 'abstract_algebra'), ('ar', 'anatomy'), ('ar', 'astronomy'), ('ar', 'business_ethics'), ('ar', 'clinical_knowledge'), ('ar', 'college_biology'), ('ar', 'college_chemistry'), ('ar', 'college_computer_science'), ('ar', 'college_mathematics'), ('ar', 'college_medicine'), ('ar', 'college_physics'), ('ar', 'computer_security'), ('ar', 'conceptual_physics'), ('ar', 'econometrics'), ('ar', 'electrical_engineering'), ('ar', 'elementary_mathematics'), ('ar', 'formal_logic'), ('ar', 'global_facts'), ('ar', 'high_school_biology'), ('ar', 'high_school_chemistry'), ('ar', 'high_school_computer_science'), ('ar', 'high_school_european_history'), ('ar', 'high_school_geography'), ('ar', 'high_school_government_and_politics'), ('ar', 'high_school_macroeconomics'), ('ar', 'high_school_mathematics'), ('ar', 'high_school_microeconomics'), ('ar', 'high_school_physics'), ('ar', 'high_school_psychology'), ('ar', 'high_school_statistics'), ('ar', 'high_school_us_history'), ('ar', 'high_school_world_history'), ('ar', 'human_aging'), ('ar', 'human_sexuality'), ('ar', 'international_law'), ('ar', 'jurisprudence'), ('ar', 'logical_fallacies'), ('ar', 'machine_learning'), ('ar', 'management'), ('ar', 'marketing'), ('ar', 'medical_genetics'), ('ar', 'miscellaneous'), ('ar', 'moral_disputes'), ('ar', 'moral_scenarios'), ('ar', 'nutrition'), ('ar', 'philosophy'), ('ar', 'prehistory'), ('ar', 'professional_accounting'), ('ar', 'professional_law'), ('ar', 'professional_medicine'), ('ar', 'professional_psychology'), ('ar', 'public_relations'), ('ar', 'security_studies'), ('ar', 'sociology'), ('ar', 'us_foreign_policy'), ('ar', 'virology'), ('ar', 'world_religions')]
LANGUAGE = {"('fr', 'abstract_algebra')": <Language.FRA: 'French'>, "('fr', 'anatomy')": <Language.FRA: 'French'>, "('fr', 'astronomy')": <Language.FRA: 'French'>, "('fr', 'business_ethics')": <Language.FRA: 'French'>, "('fr', 'clinical_knowledge')": <Language.FRA: 'French'>, "('fr', 'college_biology')": <Language.FRA: 'French'>, "('fr', 'college_chemistry')": <Language.FRA: 'French'>, "('fr', 'college_computer_science')": <Language.FRA: 'French'>, "('fr', 'college_mathematics')": <Language.FRA: 'French'>, "('fr', 'college_medicine')": <Language.FRA: 'French'>, "('fr', 'college_physics')": <Language.FRA: 'French'>, "('fr', 'computer_security')": <Language.FRA: 'French'>, "('fr', 'conceptual_physics')": <Language.FRA: 'French'>, "('fr', 'econometrics')": <Language.FRA: 'French'>, "('fr', 'electrical_engineering')": <Language.FRA: 'French'>, "('fr', 'elementary_mathematics')": <Language.FRA: 'French'>, "('fr', 'formal_logic')": <Language.FRA: 'French'>, "('fr', 'global_facts')": <Language.FRA: 'French'>, "('fr', 'high_school_biology')": <Language.FRA: 'French'>, "('fr', 'high_school_chemistry')": <Language.FRA: 'French'>, "('fr', 'high_school_computer_science')": <Language.FRA: 'French'>, "('fr', 'high_school_european_history')": <Language.FRA: 'French'>, "('fr', 'high_school_geography')": <Language.FRA: 'French'>, "('fr', 'high_school_government_and_politics')": <Language.FRA: 'French'>, "('fr', 'high_school_macroeconomics')": <Language.FRA: 'French'>, "('fr', 'high_school_mathematics')": <Language.FRA: 'French'>, "('fr', 'high_school_microeconomics')": <Language.FRA: 'French'>, "('fr', 'high_school_physics')": <Language.FRA: 'French'>, "('fr', 'high_school_psychology')": <Language.FRA: 'French'>, "('fr', 'high_school_statistics')": <Language.FRA: 'French'>, "('fr', 'high_school_us_history')": <Language.FRA: 'French'>, "('fr', 'high_school_world_history')": <Language.FRA: 'French'>, "('fr', 'human_aging')": <Language.FRA: 'French'>, "('fr', 'human_sexuality')": <Language.FRA: 'French'>, "('fr', 'international_law')": <Language.FRA: 'French'>, "('fr', 'jurisprudence')": <Language.FRA: 'French'>, "('fr', 'logical_fallacies')": <Language.FRA: 'French'>, "('fr', 'machine_learning')": <Language.FRA: 'French'>, "('fr', 'management')": <Language.FRA: 'French'>, "('fr', 'marketing')": <Language.FRA: 'French'>, "('fr', 'medical_genetics')": <Language.FRA: 'French'>, "('fr', 'miscellaneous')": <Language.FRA: 'French'>, "('fr', 'moral_disputes')": <Language.FRA: 'French'>, "('fr', 'moral_scenarios')": <Language.FRA: 'French'>, "('fr', 'nutrition')": <Language.FRA: 'French'>, "('fr', 'philosophy')": <Language.FRA: 'French'>, "('fr', 'prehistory')": <Language.FRA: 'French'>, "('fr', 'professional_accounting')": <Language.FRA: 'French'>, "('fr', 'professional_law')": <Language.FRA: 'French'>, "('fr', 'professional_medicine')": <Language.FRA: 'French'>, "('fr', 'professional_psychology')": <Language.FRA: 'French'>, "('fr', 'public_relations')": <Language.FRA: 'French'>, "('fr', 'security_studies')": <Language.FRA: 'French'>, "('fr', 'sociology')": <Language.FRA: 'French'>, "('fr', 'us_foreign_policy')": <Language.FRA: 'French'>, "('fr', 'virology')": <Language.FRA: 'French'>, "('fr', 'world_religions')": <Language.FRA: 'French'>, "('de', 'abstract_algebra')": <Language.DEU: 'German'>, "('de', 'anatomy')": <Language.DEU: 'German'>, "('de', 'astronomy')": <Language.DEU: 'German'>, "('de', 'business_ethics')": <Language.DEU: 'German'>, "('de', 'clinical_knowledge')": <Language.DEU: 'German'>, "('de', 'college_biology')": <Language.DEU: 'German'>, "('de', 'college_chemistry')": <Language.DEU: 'German'>, "('de', 'college_computer_science')": <Language.DEU: 'German'>, "('de', 'college_mathematics')": <Language.DEU: 'German'>, "('de', 'college_medicine')": <Language.DEU: 'German'>, "('de', 'college_physics')": <Language.DEU: 'German'>, "('de', 'computer_security')": <Language.DEU: 'German'>, "('de', 'conceptual_physics')": <Language.DEU: 'German'>, "('de', 'econometrics')": <Language.DEU: 'German'>, "('de', 'electrical_engineering')": <Language.DEU: 'German'>, "('de', 'elementary_mathematics')": <Language.DEU: 'German'>, "('de', 'formal_logic')": <Language.DEU: 'German'>, "('de', 'global_facts')": <Language.DEU: 'German'>, "('de', 'high_school_biology')": <Language.DEU: 'German'>, "('de', 'high_school_chemistry')": <Language.DEU: 'German'>, "('de', 'high_school_computer_science')": <Language.DEU: 'German'>, "('de', 'high_school_european_history')": <Language.DEU: 'German'>, "('de', 'high_school_geography')": <Language.DEU: 'German'>, "('de', 'high_school_government_and_politics')": <Language.DEU: 'German'>, "('de', 'high_school_macroeconomics')": <Language.DEU: 'German'>, "('de', 'high_school_mathematics')": <Language.DEU: 'German'>, "('de', 'high_school_microeconomics')": <Language.DEU: 'German'>, "('de', 'high_school_physics')": <Language.DEU: 'German'>, "('de', 'high_school_psychology')": <Language.DEU: 'German'>, "('de', 'high_school_statistics')": <Language.DEU: 'German'>, "('de', 'high_school_us_history')": <Language.DEU: 'German'>, "('de', 'high_school_world_history')": <Language.DEU: 'German'>, "('de', 'human_aging')": <Language.DEU: 'German'>, "('de', 'human_sexuality')": <Language.DEU: 'German'>, "('de', 'international_law')": <Language.DEU: 'German'>, "('de', 'jurisprudence')": <Language.DEU: 'German'>, "('de', 'logical_fallacies')": <Language.DEU: 'German'>, "('de', 'machine_learning')": <Language.DEU: 'German'>, "('de', 'management')": <Language.DEU: 'German'>, "('de', 'marketing')": <Language.DEU: 'German'>, "('de', 'medical_genetics')": <Language.DEU: 'German'>, "('de', 'miscellaneous')": <Language.DEU: 'German'>, "('de', 'moral_disputes')": <Language.DEU: 'German'>, "('de', 'moral_scenarios')": <Language.DEU: 'German'>, "('de', 'nutrition')": <Language.DEU: 'German'>, "('de', 'philosophy')": <Language.DEU: 'German'>, "('de', 'prehistory')": <Language.DEU: 'German'>, "('de', 'professional_accounting')": <Language.DEU: 'German'>, "('de', 'professional_law')": <Language.DEU: 'German'>, "('de', 'professional_medicine')": <Language.DEU: 'German'>, "('de', 'professional_psychology')": <Language.DEU: 'German'>, "('de', 'public_relations')": <Language.DEU: 'German'>, "('de', 'security_studies')": <Language.DEU: 'German'>, "('de', 'sociology')": <Language.DEU: 'German'>, "('de', 'us_foreign_policy')": <Language.DEU: 'German'>, "('de', 'virology')": <Language.DEU: 'German'>, "('de', 'world_religions')": <Language.DEU: 'German'>, "('es', 'abstract_algebra')": <Language.SPA: 'Spanish'>, "('es', 'anatomy')": <Language.SPA: 'Spanish'>, "('es', 'astronomy')": <Language.SPA: 'Spanish'>, "('es', 'business_ethics')": <Language.SPA: 'Spanish'>, "('es', 'clinical_knowledge')": <Language.SPA: 'Spanish'>, "('es', 'college_biology')": <Language.SPA: 'Spanish'>, "('es', 'college_chemistry')": <Language.SPA: 'Spanish'>, "('es', 'college_computer_science')": <Language.SPA: 'Spanish'>, "('es', 'college_mathematics')": <Language.SPA: 'Spanish'>, "('es', 'college_medicine')": <Language.SPA: 'Spanish'>, "('es', 'college_physics')": <Language.SPA: 'Spanish'>, "('es', 'computer_security')": <Language.SPA: 'Spanish'>, "('es', 'conceptual_physics')": <Language.SPA: 'Spanish'>, "('es', 'econometrics')": <Language.SPA: 'Spanish'>, "('es', 'electrical_engineering')": <Language.SPA: 'Spanish'>, "('es', 'elementary_mathematics')": <Language.SPA: 'Spanish'>, "('es', 'formal_logic')": <Language.SPA: 'Spanish'>, "('es', 'global_facts')": <Language.SPA: 'Spanish'>, "('es', 'high_school_biology')": <Language.SPA: 'Spanish'>, "('es', 'high_school_chemistry')": <Language.SPA: 'Spanish'>, "('es', 'high_school_computer_science')": <Language.SPA: 'Spanish'>, "('es', 'high_school_european_history')": <Language.SPA: 'Spanish'>, "('es', 'high_school_geography')": <Language.SPA: 'Spanish'>, "('es', 'high_school_government_and_politics')": <Language.SPA: 'Spanish'>, "('es', 'high_school_macroeconomics')": <Language.SPA: 'Spanish'>, "('es', 'high_school_mathematics')": <Language.SPA: 'Spanish'>, "('es', 'high_school_microeconomics')": <Language.SPA: 'Spanish'>, "('es', 'high_school_physics')": <Language.SPA: 'Spanish'>, "('es', 'high_school_psychology')": <Language.SPA: 'Spanish'>, "('es', 'high_school_statistics')": <Language.SPA: 'Spanish'>, "('es', 'high_school_us_history')": <Language.SPA: 'Spanish'>, "('es', 'high_school_world_history')": <Language.SPA: 'Spanish'>, "('es', 'human_aging')": <Language.SPA: 'Spanish'>, "('es', 'human_sexuality')": <Language.SPA: 'Spanish'>, "('es', 'international_law')": <Language.SPA: 'Spanish'>, "('es', 'jurisprudence')": <Language.SPA: 'Spanish'>, "('es', 'logical_fallacies')": <Language.SPA: 'Spanish'>, "('es', 'machine_learning')": <Language.SPA: 'Spanish'>, "('es', 'management')": <Language.SPA: 'Spanish'>, "('es', 'marketing')": <Language.SPA: 'Spanish'>, "('es', 'medical_genetics')": <Language.SPA: 'Spanish'>, "('es', 'miscellaneous')": <Language.SPA: 'Spanish'>, "('es', 'moral_disputes')": <Language.SPA: 'Spanish'>, "('es', 'moral_scenarios')": <Language.SPA: 'Spanish'>, "('es', 'nutrition')": <Language.SPA: 'Spanish'>, "('es', 'philosophy')": <Language.SPA: 'Spanish'>, "('es', 'prehistory')": <Language.SPA: 'Spanish'>, "('es', 'professional_accounting')": <Language.SPA: 'Spanish'>, "('es', 'professional_law')": <Language.SPA: 'Spanish'>, "('es', 'professional_medicine')": <Language.SPA: 'Spanish'>, "('es', 'professional_psychology')": <Language.SPA: 'Spanish'>, "('es', 'public_relations')": <Language.SPA: 'Spanish'>, "('es', 'security_studies')": <Language.SPA: 'Spanish'>, "('es', 'sociology')": <Language.SPA: 'Spanish'>, "('es', 'us_foreign_policy')": <Language.SPA: 'Spanish'>, "('es', 'virology')": <Language.SPA: 'Spanish'>, "('es', 'world_religions')": <Language.SPA: 'Spanish'>, "('it', 'abstract_algebra')": <Language.ITA: 'Italian'>, "('it', 'anatomy')": <Language.ITA: 'Italian'>, "('it', 'astronomy')": <Language.ITA: 'Italian'>, "('it', 'business_ethics')": <Language.ITA: 'Italian'>, "('it', 'clinical_knowledge')": <Language.ITA: 'Italian'>, "('it', 'college_biology')": <Language.ITA: 'Italian'>, "('it', 'college_chemistry')": <Language.ITA: 'Italian'>, "('it', 'college_computer_science')": <Language.ITA: 'Italian'>, "('it', 'college_mathematics')": <Language.ITA: 'Italian'>, "('it', 'college_medicine')": <Language.ITA: 'Italian'>, "('it', 'college_physics')": <Language.ITA: 'Italian'>, "('it', 'computer_security')": <Language.ITA: 'Italian'>, "('it', 'conceptual_physics')": <Language.ITA: 'Italian'>, "('it', 'econometrics')": <Language.ITA: 'Italian'>, "('it', 'electrical_engineering')": <Language.ITA: 'Italian'>, "('it', 'elementary_mathematics')": <Language.ITA: 'Italian'>, "('it', 'formal_logic')": <Language.ITA: 'Italian'>, "('it', 'global_facts')": <Language.ITA: 'Italian'>, "('it', 'high_school_biology')": <Language.ITA: 'Italian'>, "('it', 'high_school_chemistry')": <Language.ITA: 'Italian'>, "('it', 'high_school_computer_science')": <Language.ITA: 'Italian'>, "('it', 'high_school_european_history')": <Language.ITA: 'Italian'>, "('it', 'high_school_geography')": <Language.ITA: 'Italian'>, "('it', 'high_school_government_and_politics')": <Language.ITA: 'Italian'>, "('it', 'high_school_macroeconomics')": <Language.ITA: 'Italian'>, "('it', 'high_school_mathematics')": <Language.ITA: 'Italian'>, "('it', 'high_school_microeconomics')": <Language.ITA: 'Italian'>, "('it', 'high_school_physics')": <Language.ITA: 'Italian'>, "('it', 'high_school_psychology')": <Language.ITA: 'Italian'>, "('it', 'high_school_statistics')": <Language.ITA: 'Italian'>, "('it', 'high_school_us_history')": <Language.ITA: 'Italian'>, "('it', 'high_school_world_history')": <Language.ITA: 'Italian'>, "('it', 'human_aging')": <Language.ITA: 'Italian'>, "('it', 'human_sexuality')": <Language.ITA: 'Italian'>, "('it', 'international_law')": <Language.ITA: 'Italian'>, "('it', 'jurisprudence')": <Language.ITA: 'Italian'>, "('it', 'logical_fallacies')": <Language.ITA: 'Italian'>, "('it', 'machine_learning')": <Language.ITA: 'Italian'>, "('it', 'management')": <Language.ITA: 'Italian'>, "('it', 'marketing')": <Language.ITA: 'Italian'>, "('it', 'medical_genetics')": <Language.ITA: 'Italian'>, "('it', 'miscellaneous')": <Language.ITA: 'Italian'>, "('it', 'moral_disputes')": <Language.ITA: 'Italian'>, "('it', 'moral_scenarios')": <Language.ITA: 'Italian'>, "('it', 'nutrition')": <Language.ITA: 'Italian'>, "('it', 'philosophy')": <Language.ITA: 'Italian'>, "('it', 'prehistory')": <Language.ITA: 'Italian'>, "('it', 'professional_accounting')": <Language.ITA: 'Italian'>, "('it', 'professional_law')": <Language.ITA: 'Italian'>, "('it', 'professional_medicine')": <Language.ITA: 'Italian'>, "('it', 'professional_psychology')": <Language.ITA: 'Italian'>, "('it', 'public_relations')": <Language.ITA: 'Italian'>, "('it', 'security_studies')": <Language.ITA: 'Italian'>, "('it', 'sociology')": <Language.ITA: 'Italian'>, "('it', 'us_foreign_policy')": <Language.ITA: 'Italian'>, "('it', 'virology')": <Language.ITA: 'Italian'>, "('it', 'world_religions')": <Language.ITA: 'Italian'>, "('pt', 'abstract_algebra')": <Language.POR: 'Portuguese'>, "('pt', 'anatomy')": <Language.POR: 'Portuguese'>, "('pt', 'astronomy')": <Language.POR: 'Portuguese'>, "('pt', 'business_ethics')": <Language.POR: 'Portuguese'>, "('pt', 'clinical_knowledge')": <Language.POR: 'Portuguese'>, "('pt', 'college_biology')": <Language.POR: 'Portuguese'>, "('pt', 'college_chemistry')": <Language.POR: 'Portuguese'>, "('pt', 'college_computer_science')": <Language.POR: 'Portuguese'>, "('pt', 'college_mathematics')": <Language.POR: 'Portuguese'>, "('pt', 'college_medicine')": <Language.POR: 'Portuguese'>, "('pt', 'college_physics')": <Language.POR: 'Portuguese'>, "('pt', 'computer_security')": <Language.POR: 'Portuguese'>, "('pt', 'conceptual_physics')": <Language.POR: 'Portuguese'>, "('pt', 'econometrics')": <Language.POR: 'Portuguese'>, "('pt', 'electrical_engineering')": <Language.POR: 'Portuguese'>, "('pt', 'elementary_mathematics')": <Language.POR: 'Portuguese'>, "('pt', 'formal_logic')": <Language.POR: 'Portuguese'>, "('pt', 'global_facts')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_biology')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_chemistry')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_computer_science')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_european_history')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_geography')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_government_and_politics')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_macroeconomics')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_mathematics')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_microeconomics')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_physics')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_psychology')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_statistics')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_us_history')": <Language.POR: 'Portuguese'>, "('pt', 'high_school_world_history')": <Language.POR: 'Portuguese'>, "('pt', 'human_aging')": <Language.POR: 'Portuguese'>, "('pt', 'human_sexuality')": <Language.POR: 'Portuguese'>, "('pt', 'international_law')": <Language.POR: 'Portuguese'>, "('pt', 'jurisprudence')": <Language.POR: 'Portuguese'>, "('pt', 'logical_fallacies')": <Language.POR: 'Portuguese'>, "('pt', 'machine_learning')": <Language.POR: 'Portuguese'>, "('pt', 'management')": <Language.POR: 'Portuguese'>, "('pt', 'marketing')": <Language.POR: 'Portuguese'>, "('pt', 'medical_genetics')": <Language.POR: 'Portuguese'>, "('pt', 'miscellaneous')": <Language.POR: 'Portuguese'>, "('pt', 'moral_disputes')": <Language.POR: 'Portuguese'>, "('pt', 'moral_scenarios')": <Language.POR: 'Portuguese'>, "('pt', 'nutrition')": <Language.POR: 'Portuguese'>, "('pt', 'philosophy')": <Language.POR: 'Portuguese'>, "('pt', 'prehistory')": <Language.POR: 'Portuguese'>, "('pt', 'professional_accounting')": <Language.POR: 'Portuguese'>, "('pt', 'professional_law')": <Language.POR: 'Portuguese'>, "('pt', 'professional_medicine')": <Language.POR: 'Portuguese'>, "('pt', 'professional_psychology')": <Language.POR: 'Portuguese'>, "('pt', 'public_relations')": <Language.POR: 'Portuguese'>, "('pt', 'security_studies')": <Language.POR: 'Portuguese'>, "('pt', 'sociology')": <Language.POR: 'Portuguese'>, "('pt', 'us_foreign_policy')": <Language.POR: 'Portuguese'>, "('pt', 'virology')": <Language.POR: 'Portuguese'>, "('pt', 'world_religions')": <Language.POR: 'Portuguese'>, "('ar', 'abstract_algebra')": <Language.ARB: 'Arabic'>, "('ar', 'anatomy')": <Language.ARB: 'Arabic'>, "('ar', 'astronomy')": <Language.ARB: 'Arabic'>, "('ar', 'business_ethics')": <Language.ARB: 'Arabic'>, "('ar', 'clinical_knowledge')": <Language.ARB: 'Arabic'>, "('ar', 'college_biology')": <Language.ARB: 'Arabic'>, "('ar', 'college_chemistry')": <Language.ARB: 'Arabic'>, "('ar', 'college_computer_science')": <Language.ARB: 'Arabic'>, "('ar', 'college_mathematics')": <Language.ARB: 'Arabic'>, "('ar', 'college_medicine')": <Language.ARB: 'Arabic'>, "('ar', 'college_physics')": <Language.ARB: 'Arabic'>, "('ar', 'computer_security')": <Language.ARB: 'Arabic'>, "('ar', 'conceptual_physics')": <Language.ARB: 'Arabic'>, "('ar', 'econometrics')": <Language.ARB: 'Arabic'>, "('ar', 'electrical_engineering')": <Language.ARB: 'Arabic'>, "('ar', 'elementary_mathematics')": <Language.ARB: 'Arabic'>, "('ar', 'formal_logic')": <Language.ARB: 'Arabic'>, "('ar', 'global_facts')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_biology')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_chemistry')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_computer_science')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_european_history')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_geography')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_government_and_politics')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_macroeconomics')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_mathematics')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_microeconomics')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_physics')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_psychology')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_statistics')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_us_history')": <Language.ARB: 'Arabic'>, "('ar', 'high_school_world_history')": <Language.ARB: 'Arabic'>, "('ar', 'human_aging')": <Language.ARB: 'Arabic'>, "('ar', 'human_sexuality')": <Language.ARB: 'Arabic'>, "('ar', 'international_law')": <Language.ARB: 'Arabic'>, "('ar', 'jurisprudence')": <Language.ARB: 'Arabic'>, "('ar', 'logical_fallacies')": <Language.ARB: 'Arabic'>, "('ar', 'machine_learning')": <Language.ARB: 'Arabic'>, "('ar', 'management')": <Language.ARB: 'Arabic'>, "('ar', 'marketing')": <Language.ARB: 'Arabic'>, "('ar', 'medical_genetics')": <Language.ARB: 'Arabic'>, "('ar', 'miscellaneous')": <Language.ARB: 'Arabic'>, "('ar', 'moral_disputes')": <Language.ARB: 'Arabic'>, "('ar', 'moral_scenarios')": <Language.ARB: 'Arabic'>, "('ar', 'nutrition')": <Language.ARB: 'Arabic'>, "('ar', 'philosophy')": <Language.ARB: 'Arabic'>, "('ar', 'prehistory')": <Language.ARB: 'Arabic'>, "('ar', 'professional_accounting')": <Language.ARB: 'Arabic'>, "('ar', 'professional_law')": <Language.ARB: 'Arabic'>, "('ar', 'professional_medicine')": <Language.ARB: 'Arabic'>, "('ar', 'professional_psychology')": <Language.ARB: 'Arabic'>, "('ar', 'public_relations')": <Language.ARB: 'Arabic'>, "('ar', 'security_studies')": <Language.ARB: 'Arabic'>, "('ar', 'sociology')": <Language.ARB: 'Arabic'>, "('ar', 'us_foreign_policy')": <Language.ARB: 'Arabic'>, "('ar', 'virology')": <Language.ARB: 'Arabic'>, "('ar', 'world_religions')": <Language.ARB: 'Arabic'>}

More detailed documentation, with prompt examples and ground truth completions, can be generated with uv run -m eval_framework.utils.generate_task_docs --add-prompt-examples --only-tasks "GlobalMMLU".