Index A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W A AccuracyBayesianLoglikelihood (class in eval_framework.metrics.loglikelihood.accuracy_loglikelihood) AccuracyCompletion (class in eval_framework.metrics.completion.accuracy_completion) AccuracyCompletionOLMES (class in eval_framework.metrics.completion.accuracy_completion) AccuracyCompletionWithEvaluate (class in eval_framework.metrics.completion.accuracy_completion) AccuracyLoglikelihood (class in eval_framework.metrics.loglikelihood.accuracy_loglikelihood) AccuracyNormLoglikelihood (class in eval_framework.metrics.loglikelihood.accuracy_loglikelihood) add() (eval_framework.tasks.registry.Registry method) additional_kwargs (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) AffineFit (class in eval_framework.metrics.loglikelihood.bpb_variants_estimators) aggregate_prior_bpb_metrics() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) aggregates (eval_framework.suite.SuiteResult attribute) (eval_framework.suite.TaskSuite attribute) Aggregator (class in eval_framework.metrics.aggregators.aggregators) AGGREGATORS (eval_framework.metrics.base.BaseMetric attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletion attribute) aime() (in module eval_framework.benchmarks.math_reasoning) aime2024() (in module eval_framework.benchmarks.math_reasoning) answer (eval_framework.fewshot.FewshotExample attribute) answer_key_to_index() (in module eval_framework.tasks.task_style) answer_label (eval_framework.benchmarks.utils.global_mmlu_languages.GlobalMMLULocale attribute) answer_tuples (eval_framework.metrics.completion.drop_completion.DropMetricContext attribute) AnswerPolicy (class in eval_framework.answer) arc() (in module eval_framework.benchmarks.arc) arc_de() (in module eval_framework.benchmarks.arc_de) arc_ellamind() (in module eval_framework.benchmarks.arc_ellamind) arc_idk() (in module eval_framework.benchmarks.arc) arc_olmes() (in module eval_framework.benchmarks.arc) ArcDeReader (class in eval_framework.benchmarks.arc_de) ArcEllamindReader (class in eval_framework.benchmarks.arc_ellamind) ArcReader (class in eval_framework.benchmarks.arc) artifact_upload_function() (in module eval_framework.result_processors.wandb_uploader) as_dict() (eval_framework.base_config.BaseConfig method) assemble_messages() (in module eval_framework.eval_kind) attempts (eval_framework.llm.retry.RetryPolicy attribute) B backoff_s (eval_framework.llm.retry.RetryPolicy attribute) BaseConfig (class in eval_framework.base_config) BaseLLM (class in eval_framework.llm.base) BaseLLMJudgeMetric (class in eval_framework.metrics.llm.base) BaseLoglikelihoodMetric (class in eval_framework.metrics.loglikelihood.base) BaseMetric (class in eval_framework.metrics.base) batch_size (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) Benchmark (class in eval_framework.contract) (class in eval_framework.tasks.registry) benchmark_timeout (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) bigcodebench_olmes() (in module eval_framework.benchmarks.bigcodebench) bin_mass() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) bind() (eval_framework.fewshot.FewShot method) (eval_framework.fewshot.FewShotPolicy method) (eval_framework.fewshot.NoFewShot method) bits (eval_framework.metrics.loglikelihood.bpb_variants_estimators.PrefixItem attribute) bits_per_answer() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) BitsPerByteLoglikelihood (class in eval_framework.metrics.loglikelihood.bits_per_byte) BitsPerByteVariantsLoglikelihood (class in eval_framework.metrics.loglikelihood.bpb_variants) BPB (eval_framework.tasks.task_style.TaskStyle attribute) bpb() (in module eval_framework.benchmarks.humaneval) (in module eval_framework.benchmarks.mbpp) bpb_at_nstar() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) BPBStyle (class in eval_framework.tasks.task_style) build_pool() (eval_framework.fewshot.FewShotSource method) (eval_framework.fewshot.FewShotSplit method) (eval_framework.fewshot.Predefined method) (eval_framework.fewshot.SampleSplit method) build_prefix_item() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) byte_lens (eval_framework.metrics.loglikelihood.bpb_variants_estimators.PrefixItem attribute) BYTES_PER_TOKEN (eval_framework.llm.openai.OpenAIModel attribute) BytesCompletion (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) BytesLoglikelihood (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) C calculate() (eval_framework.metrics.base.BaseMetric method) (eval_framework.metrics.completion.accuracy_completion.AccuracyCompletion method) (eval_framework.metrics.completion.accuracy_completion.AccuracyCompletionWithEvaluate method) (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion method) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOne method) (eval_framework.metrics.completion.concordance_index.ConcordanceIndex method) (eval_framework.metrics.completion.csv_format.CSVFormat method) (eval_framework.metrics.completion.drop_completion.DropF1ExactMatch method) (eval_framework.metrics.completion.exponential_similarity.ExponentialSimilarity method) (eval_framework.metrics.completion.f1.F1 method) (eval_framework.metrics.completion.format_checker.CheckJsonFormat method) (eval_framework.metrics.completion.format_checker.CheckPostScriptFormat method) (eval_framework.metrics.completion.grid_difference.GridDifference method) (eval_framework.metrics.completion.ifeval.IFEvalMetric method) (eval_framework.metrics.completion.json_format.JsonFormat method) (eval_framework.metrics.completion.language_checker.GermanCompletionChecker method) (eval_framework.metrics.completion.language_checker.LanguageChecker method) (eval_framework.metrics.completion.language_checker.LanguageConsistencyChecker method) (eval_framework.metrics.completion.language_checker.LanguageRawConsistencyChecker method) (eval_framework.metrics.completion.length_control.LengthControl method) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletion method) (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion method) (eval_framework.metrics.completion.multipl_e_assertion.MultiPLECodeAssertion method) (eval_framework.metrics.completion.placeholder_checker.PlaceholderChecker method) (eval_framework.metrics.completion.repetition.WordRepetition method) (eval_framework.metrics.completion.rouge_1.ROUGE_1 method) (eval_framework.metrics.completion.rouge_2.ROUGE_2 method) (eval_framework.metrics.completion.rouge_geometric_mean.ROUGE_GEOMETRIC_MEAN method) (eval_framework.metrics.completion.rouge_l.ROUGE_L method) (eval_framework.metrics.completion.text_counter.ParagraphCounter method) (eval_framework.metrics.completion.text_counter.ResponseToOriginalLengthRatio method) (eval_framework.metrics.completion.text_counter.SentenceCounter method) (eval_framework.metrics.completion.text_counter.WordCounter method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesCompletion method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesLoglikelihood method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsCompletion method) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsLoglikelihood method) (eval_framework.metrics.efficiency.finish_reason.FinishReason method) (eval_framework.metrics.efficiency.token_counters.TokenCounts method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyBayesianLoglikelihood method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyLoglikelihood method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyNormLoglikelihood method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.PartialEvalAccuracy method) (eval_framework.metrics.loglikelihood.bits_per_byte.BitsPerByteLoglikelihood method) (eval_framework.metrics.loglikelihood.bpb_variants.BitsPerByteVariantsLoglikelihood method) (eval_framework.metrics.loglikelihood.bpb_variants_instrumented.InstrumentedBitsPerByte method) (eval_framework.metrics.loglikelihood.bpb_variants_prefix.PrefixBitsPerByte method) (eval_framework.metrics.loglikelihood.confidence_weighted_accuracy.ConfidenceWeightedAccuracy method) (eval_framework.metrics.loglikelihood.dcs.DistributionalCorrectnessScore method) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMass method) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMassNorm method) (eval_framework.metrics.loglikelihood.ternary.TernaryScore method) calculate_concordance_index() (in module eval_framework.metrics.completion.concordance_index) calculate_exponential_similarity() (in module eval_framework.metrics.completion.exponential_similarity) calculate_f1() (in module eval_framework.metrics.completion.f1) calculate_score() (eval_framework.metrics.completion.grid_difference.GridDifference method) CallableSerializer (class in eval_framework.tasks.utils) category_names (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale attribute) CES (eval_framework.tasks.task_style.Language attribute) check_for_equation() (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion method) CheckJsonFormat (class in eval_framework.metrics.completion.format_checker) CheckPostScriptFormat (class in eval_framework.metrics.completion.format_checker) child (eval_framework.suite.MetricSource attribute) Choice (class in eval_framework.eval_kind) choice() (eval_framework.composed.ComposedBenchmark class method) ChoiceFields (class in eval_framework.choices) ChoiceReader (class in eval_framework.choices) ChoiceRenderer (class in eval_framework.fewshot) choices (eval_framework.choices.ChoiceFields attribute) classproperty (class in eval_framework.metrics.base) clean_short_answer() (in module eval_framework.benchmarks.gsm8k) close_pools() (in module eval_framework.tasks.utils) closed_form_passatk() (in module eval_framework.metrics.aggregators.aggregators) CLOZE (eval_framework.tasks.task_style.TaskStyle attribute) cloze_preamble() (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale method) cloze_template (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale attribute) ClozeStyle (class in eval_framework.tasks.task_style) code_execution_trace (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) code_prompt (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) CodeCompletionAssertion (class in eval_framework.metrics.completion.code_assertion) CodeExecutionBaseContext (class in eval_framework.metrics.completion.code_execution_pass_at_one) CodeExecutionPassAtOne (class in eval_framework.metrics.completion.code_execution_pass_at_one) CodeExecutionPassAtOneContext (class in eval_framework.metrics.completion.code_execution_pass_at_one) CodeExecutionPassAtOneWithCodebench (class in eval_framework.metrics.completion.code_execution_pass_at_one) CodeReconstructor (class in eval_framework.answer) collect_prefix_items() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) CommonsenseQaReader (class in eval_framework.benchmarks.csqa) comparison (eval_framework.metrics.completion.text_counter.ParagraphCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.SentenceCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.WordCounterMetricContext attribute) COMPLETION (eval_framework.contract.ResponseType attribute) compose() (eval_framework.composed.ComposedBenchmark class method) ComposedBenchmark (class in eval_framework.composed) ComposedEval (class in eval_framework.composed) compute_aggregates() (in module eval_framework.suite) compute_all_bpb_results() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) compute_prefix_bpb_results() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) (in module eval_framework.metrics.loglikelihood.bpb_variants_prefix) compute_prior_bpb_for_items() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) compute_standard_bpb_results() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) (in module eval_framework.metrics.loglikelihood.bpb_variants_instrumented) ConcordanceIndex (class in eval_framework.metrics.completion.concordance_index) ConfidenceWeightedAccuracy (class in eval_framework.metrics.loglikelihood.confidence_weighted_accuracy) config (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale attribute) (eval_framework.benchmarks.utils.ellamind_languages.EllamindLanguage attribute) content_bytes (eval_framework.metrics.loglikelihood.bpb_variants_estimators.PrefixItem property) context (eval_framework.contract.ResultCandidate attribute) (eval_framework.eval_kind.SampleBody attribute) continuation_rate() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) copa_mc_olmes() (in module eval_framework.benchmarks.copa) CopaReader (class in eval_framework.benchmarks.copa) corpus_bpb() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) correct_index (eval_framework.choices.ChoiceFields attribute) Cot (class in eval_framework.benchmarks.cot) cot_style (eval_framework.benchmarks.hendrycks_math_ellamind.MinervaLocale attribute) COT_STYLE (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletion attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionDE attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionFR attribute) count_differences() (eval_framework.metrics.completion.grid_difference.GridDifference method) create() (eval_framework.composed.ComposedBenchmark method) (eval_framework.contract.Benchmark method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) csqa_ellamind_bpb_de() (in module eval_framework.benchmarks.csqa_ellamind) csqa_ellamind_cloze_easy_de() (in module eval_framework.benchmarks.csqa_ellamind) csqa_ellamind_cloze_hard_de() (in module eval_framework.benchmarks.csqa_ellamind) csqa_ellamind_mc_easy_de() (in module eval_framework.benchmarks.csqa_ellamind) csqa_ellamind_mc_hard_de() (in module eval_framework.benchmarks.csqa_ellamind) csqa_mc_olmes() (in module eval_framework.benchmarks.csqa) CsqaReader (class in eval_framework.benchmarks.csqa_ellamind) CSVFormat (class in eval_framework.metrics.completion.csv_format) CSVFormatEvaluation (class in eval_framework.metrics.completion.csv_format) cue (eval_framework.eval_kind.SampleBody attribute) cumulative_cost() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) D DAN (eval_framework.tasks.task_style.Language attribute) DatasetLoader (class in eval_framework.tasks.dataset_loading) DatasetPolicy (class in eval_framework.tasks.dataset_loading) decode() (eval_framework.tasks.utils.CallableSerializer static method) DecodingError DEFAULT_FORMATTER (eval_framework.llm.openai.OpenAIModel attribute) delete_output_dir_after_upload (eval_framework.tasks.eval_config.EvalConfig attribute) description (eval_framework.tasks.eval_config.EvalConfig attribute) DEU (eval_framework.tasks.task_style.Language attribute) display_name() (eval_framework.composed.ComposedBenchmark method) (eval_framework.composed.ComposedEval method) (eval_framework.contract.Benchmark method) (eval_framework.contract.Eval method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) DistributionalCorrectnessScore (class in eval_framework.metrics.loglikelihood.dcs) DockerReturnedEmptyOutput documentation() (eval_framework.benchmarks.mmlu_prox.MMLUProXDatasetPolicy method) (eval_framework.fewshot.FewShot method) (eval_framework.fewshot.FewShotPolicy method) (eval_framework.fewshot.FewShotSource method) (eval_framework.fewshot.FewShotSplit method) (eval_framework.fewshot.NoFewShot method) (eval_framework.fewshot.Predefined method) (eval_framework.fewshot.SampleSplit method) (eval_framework.tasks.dataset_loading.DatasetPolicy method) (eval_framework.tasks.dataset_loading.SubjectColumn method) (eval_framework.tasks.dataset_loading.Subset method) (eval_framework.tasks.dataset_revisions.FixedHfConfig method) (eval_framework.tasks.dataset_revisions.Pinned method) draw() (eval_framework.fewshot.FewShotSource method) (eval_framework.fewshot.FewShotSplit method) (eval_framework.fewshot.Predefined method) (eval_framework.fewshot.SampleSplit method) drop_completion_olmes() (in module eval_framework.benchmarks.drop) drop_mc_olmes() (in module eval_framework.benchmarks.drop) DropF1ExactMatch (class in eval_framework.metrics.completion.drop_completion) DropMetricContext (class in eval_framework.metrics.completion.drop_completion) E effective_length_bpb() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) ellamind_language() (in module eval_framework.benchmarks.utils.ellamind_languages) EllamindLanguage (class in eval_framework.benchmarks.utils.ellamind_languages) EmptyCompletionError encode() (eval_framework.tasks.utils.CallableSerializer static method) EncodingError ENG (eval_framework.tasks.task_style.Language attribute) entry_cost() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) entry_point (eval_framework.benchmarks.humaneval.HumanEvalMetricContext attribute) error (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) estimate_pass_at_k() (in module eval_framework.metrics.completion.code_execution_pass_at_one) Eval (class in eval_framework.contract) eval_framework module eval_framework.answer module eval_framework.base_config module eval_framework.benchmarks module eval_framework.benchmarks.arc module eval_framework.benchmarks.arc_de module eval_framework.benchmarks.arc_ellamind module eval_framework.benchmarks.bigcodebench module eval_framework.benchmarks.copa module eval_framework.benchmarks.cot module eval_framework.benchmarks.csqa module eval_framework.benchmarks.csqa_ellamind module eval_framework.benchmarks.drop module eval_framework.benchmarks.global_mmlu module eval_framework.benchmarks.goldenswag module eval_framework.benchmarks.gpqa module eval_framework.benchmarks.gpqa_ellamind module eval_framework.benchmarks.gsm8k module eval_framework.benchmarks.gsm8k_ellamind module eval_framework.benchmarks.hellaswag module eval_framework.benchmarks.hellaswag_ellamind module eval_framework.benchmarks.hendrycks_math_ellamind module eval_framework.benchmarks.hle_ellamind module eval_framework.benchmarks.humaneval module eval_framework.benchmarks.humaneval_ellamind module eval_framework.benchmarks.humaneval_plus module eval_framework.benchmarks.ifeval module eval_framework.benchmarks.lbpp module eval_framework.benchmarks.math_reasoning module eval_framework.benchmarks.mbpp module eval_framework.benchmarks.mbpp_ellamind module eval_framework.benchmarks.mmlu module eval_framework.benchmarks.mmlu_pro module eval_framework.benchmarks.mmlu_prox module eval_framework.benchmarks.multipl_e module eval_framework.benchmarks.naturalqs_open module eval_framework.benchmarks.piqa module eval_framework.benchmarks.piqa_ellamind module eval_framework.benchmarks.sciq module eval_framework.benchmarks.simpleqa_ellamind module eval_framework.benchmarks.siqa_ellamind module eval_framework.benchmarks.social_iqa module eval_framework.benchmarks.squad module eval_framework.benchmarks.utils module eval_framework.benchmarks.utils.ellamind_languages module eval_framework.benchmarks.utils.global_mmlu_languages module eval_framework.benchmarks.winogrande module eval_framework.benchmarks.winogrande_ellamind module eval_framework.choices module eval_framework.composed module eval_framework.contract module eval_framework.eval_kind module eval_framework.evaluation_generator module eval_framework.exceptions module eval_framework.fewshot module eval_framework.llm module eval_framework.llm.base module eval_framework.llm.models module eval_framework.llm.openai module eval_framework.llm.retry module eval_framework.logger module eval_framework.main module eval_framework.metrics module eval_framework.metrics.aggregators module eval_framework.metrics.aggregators.aggregators module eval_framework.metrics.base module eval_framework.metrics.completion module eval_framework.metrics.completion.accuracy_completion module eval_framework.metrics.completion.code_assertion module eval_framework.metrics.completion.code_execution_pass_at_one module eval_framework.metrics.completion.concordance_index module eval_framework.metrics.completion.csv_format module eval_framework.metrics.completion.drop_completion module eval_framework.metrics.completion.exponential_similarity module eval_framework.metrics.completion.f1 module eval_framework.metrics.completion.format_checker module eval_framework.metrics.completion.grid_difference module eval_framework.metrics.completion.ifeval module eval_framework.metrics.completion.json_format module eval_framework.metrics.completion.language_checker module eval_framework.metrics.completion.length_control module eval_framework.metrics.completion.math_minerva_completion module eval_framework.metrics.completion.math_reasoning_completion module eval_framework.metrics.completion.minerva_math_utils module eval_framework.metrics.completion.multipl_e_assertion module eval_framework.metrics.completion.placeholder_checker module eval_framework.metrics.completion.repetition module eval_framework.metrics.completion.rouge_1 module eval_framework.metrics.completion.rouge_2 module eval_framework.metrics.completion.rouge_geometric_mean module eval_framework.metrics.completion.rouge_l module eval_framework.metrics.completion.text_counter module eval_framework.metrics.efficiency module eval_framework.metrics.efficiency.bytes_per_sequence_position module eval_framework.metrics.efficiency.finish_reason module eval_framework.metrics.efficiency.token_counters module eval_framework.metrics.llm module eval_framework.metrics.llm.base module eval_framework.metrics.loglikelihood module eval_framework.metrics.loglikelihood.accuracy_loglikelihood module eval_framework.metrics.loglikelihood.base module eval_framework.metrics.loglikelihood.bits_per_byte module eval_framework.metrics.loglikelihood.bpb_variants module eval_framework.metrics.loglikelihood.bpb_variants_common module eval_framework.metrics.loglikelihood.bpb_variants_estimators module eval_framework.metrics.loglikelihood.bpb_variants_instrumented module eval_framework.metrics.loglikelihood.bpb_variants_prefix module eval_framework.metrics.loglikelihood.confidence_weighted_accuracy module eval_framework.metrics.loglikelihood.dcs module eval_framework.metrics.loglikelihood.probability_mass module eval_framework.metrics.loglikelihood.ternary module eval_framework.response_generator module eval_framework.result_processors module eval_framework.result_processors.base module eval_framework.result_processors.hf_uploader module eval_framework.result_processors.result_processor module eval_framework.result_processors.wandb_uploader module eval_framework.run module eval_framework.run_direct module eval_framework.subjects module eval_framework.suite module eval_framework.tasks module eval_framework.tasks.base module eval_framework.tasks.dataset_loading module eval_framework.tasks.dataset_revisions module eval_framework.tasks.eval_config module eval_framework.tasks.lazy module eval_framework.tasks.markdown_doc module eval_framework.tasks.registry module eval_framework.tasks.task_loader module eval_framework.tasks.task_names module eval_framework.tasks.task_style module eval_framework.tasks.utils module EvalConfig (class in eval_framework.tasks.eval_config) EvalKind (class in eval_framework.eval_kind) evalplus_cue (eval_framework.benchmarks.mbpp_ellamind.MbppLocale attribute) evalplus_reconstruct() (in module eval_framework.benchmarks.mbpp) evaluate_csv_format() (in module eval_framework.metrics.completion.csv_format) EvaluationGenerator (class in eval_framework.evaluation_generator) exact_match (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) exact_match_hf_evaluate() (in module eval_framework.metrics.completion.accuracy_completion) example_shots (eval_framework.fewshot.FewShotDoc attribute) execute_python_code_with_tests() (in module eval_framework.tasks.utils) execution() (in module eval_framework.benchmarks.humaneval) (in module eval_framework.benchmarks.mbpp) ExecutionResult (class in eval_framework.tasks.utils) ExponentialSimilarity (class in eval_framework.metrics.completion.exponential_similarity) extra_llm_args (eval_framework.suite.TaskSuite attribute) extract_answer() (eval_framework.answer.AnswerPolicy method) (eval_framework.answer.ExtractFromCompletion method) (eval_framework.answer.PickFromCandidates method) (eval_framework.answer.ReconstructProgram method) extract_answers() (in module eval_framework.metrics.completion.minerva_math_utils) extract_csv_from_text() (in module eval_framework.metrics.completion.csv_format) extract_grid_from_prompt() (eval_framework.metrics.completion.grid_difference.GridDifference method) extract_hash_answer() (in module eval_framework.benchmarks.math_reasoning) extract_imports() (in module eval_framework.tasks.utils) extract_python_code_from_response() (in module eval_framework.tasks.utils) ExtractFromCompletion (class in eval_framework.answer) F F1 (class in eval_framework.metrics.completion.f1) F1SquadNormalized (class in eval_framework.metrics.completion.f1) FewShot (class in eval_framework.fewshot) FewShotDoc (class in eval_framework.fewshot) FewshotExample (class in eval_framework.fewshot) FewShotGenerator (class in eval_framework.fewshot) FewShotPolicy (class in eval_framework.fewshot) FewShotRenderer (class in eval_framework.fewshot) FewShotSource (class in eval_framework.fewshot) FewShotSplit (class in eval_framework.fewshot) FIN (eval_framework.tasks.task_style.Language attribute) final_answer_line (eval_framework.benchmarks.hendrycks_math_ellamind.MinervaLocale attribute) FinishReason (class in eval_framework.metrics.efficiency.finish_reason) first_match() (in module eval_framework.answer) FixedHfConfig (class in eval_framework.tasks.dataset_revisions) FixedHfConfigLoader (class in eval_framework.tasks.dataset_loading) for_item() (eval_framework.fewshot.FewShotGenerator method) for_language() (eval_framework.tasks.task_style.TaskStyler class method) format_mc_prompt() (in module eval_framework.tasks.task_style) FRA (eval_framework.tasks.task_style.Language attribute) from_context() (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext class method) from_file() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions class method) from_yaml() (eval_framework.base_config.BaseConfig class method) fulfills_schema (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) FunctionRenderer (class in eval_framework.fewshot) G generate() (eval_framework.llm.base.BaseLLM method) (eval_framework.response_generator.ResponseGenerator method) generate_completions() (eval_framework.composed.ComposedEval method) (eval_framework.contract.Eval method) generate_embeddings() (eval_framework.llm.openai.OpenAIEmbeddingModel method) generate_from_messages() (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.openai.OpenAIEmbeddingModel method) (eval_framework.llm.openai.OpenAIModel method) (eval_framework.llm.retry.RetryingLLM method) generate_from_samples() (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.retry.RetryingLLM method) generate_output_dir() (in module eval_framework.result_processors.result_processor) Generative (class in eval_framework.eval_kind) geometric_survival() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) GermanCompletionChecker (class in eval_framework.metrics.completion.language_checker) get_cue_text() (eval_framework.tasks.task_style.ClozeStyle method) (eval_framework.tasks.task_style.IdkStyle method) (eval_framework.tasks.task_style.MCCompletionStyle method) (eval_framework.tasks.task_style.MCStyle method) (eval_framework.tasks.task_style.TaskStyler method) get_docker_address() (in module eval_framework.tasks.utils) get_external_dependencies() (in module eval_framework.tasks.utils) get_extra_metadata() (eval_framework.tasks.task_style.TaskStyler method) get_fewshot_target_text() (eval_framework.tasks.task_style.MCCompletionStyle method) (eval_framework.tasks.task_style.TaskStyler method) get_ground_truth() (eval_framework.tasks.task_style.ClozeStyle method) (eval_framework.tasks.task_style.IdkStyle method) (eval_framework.tasks.task_style.MCCompletionStyle method) (eval_framework.tasks.task_style.MCStyle method) (eval_framework.tasks.task_style.TaskStyler method) get_hyperparam_overrides() (eval_framework.suite.TaskSuite method) get_instruction_text() (eval_framework.tasks.task_style.ClozeStyle method) (eval_framework.tasks.task_style.IdkStyle method) (eval_framework.tasks.task_style.MCCompletionStyle method) (eval_framework.tasks.task_style.MCStyle method) (eval_framework.tasks.task_style.TaskStyler method) get_json_object() (in module eval_framework.metrics.completion.json_format) get_max_tokens() (eval_framework.composed.ComposedEval method) (eval_framework.contract.Eval method) get_metadata() (eval_framework.composed.ComposedEval method) (eval_framework.contract.Eval method) get_n_letters() (in module eval_framework.tasks.utils) get_or_create_pool() (in module eval_framework.tasks.utils) get_possible_completions() (eval_framework.tasks.task_style.BPBStyle method) (eval_framework.tasks.task_style.ClozeStyle method) (eval_framework.tasks.task_style.IdkStyle method) (eval_framework.tasks.task_style.MCCompletionStyle method) (eval_framework.tasks.task_style.MCStyle method) (eval_framework.tasks.task_style.TaskStyler method) get_question_text() (eval_framework.tasks.task_style.TaskStyler method) get_response_type() (eval_framework.composed.ComposedEval method) (eval_framework.contract.Eval method) get_stop_sequences() (eval_framework.composed.ComposedEval method) (eval_framework.contract.Eval method) get_unnormalized_answer() (in module eval_framework.metrics.completion.minerva_math_utils) get_unnormalized_answer_de() (in module eval_framework.metrics.completion.minerva_math_utils) get_unnormalized_answer_fr() (in module eval_framework.metrics.completion.minerva_math_utils) global_mmlu() (in module eval_framework.benchmarks.global_mmlu) global_mmlu_german() (in module eval_framework.benchmarks.global_mmlu) GlobalMMLULocale (class in eval_framework.benchmarks.utils.global_mmlu_languages) gold (eval_framework.benchmarks.humaneval.SingleGoldReader attribute) (eval_framework.benchmarks.mbpp.SingleGoldReader attribute) goldenswag() (in module eval_framework.benchmarks.goldenswag) goldenswag_idk() (in module eval_framework.benchmarks.goldenswag) gpqa_diamond_cot() (in module eval_framework.benchmarks.gpqa) gpqa_diamond_cot_v2() (in module eval_framework.benchmarks.gpqa) gpqa_ellamind_bpb_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_ellamind_cloze_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_ellamind_diamond_bpb_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_ellamind_diamond_cloze_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_ellamind_diamond_cot_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_ellamind_diamond_mc_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_ellamind_mc_de() (in module eval_framework.benchmarks.gpqa_ellamind) gpqa_olmes() (in module eval_framework.benchmarks.gpqa) GpqaReader (class in eval_framework.benchmarks.gpqa) (class in eval_framework.benchmarks.gpqa_ellamind) GridDifference (class in eval_framework.metrics.completion.grid_difference) ground_truth (eval_framework.contract.ResultCandidate attribute) (eval_framework.eval_kind.SampleBody attribute) gsm8k_bpb() (in module eval_framework.benchmarks.gsm8k) gsm8k_ellamind_bpb_platinum() (in module eval_framework.benchmarks.gsm8k_ellamind) gsm8k_ellamind_platinum() (in module eval_framework.benchmarks.gsm8k_ellamind) gsm8k_olmes() (in module eval_framework.benchmarks.gsm8k) gsm8k_reasoning() (in module eval_framework.benchmarks.math_reasoning) H has_csv (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) hellaswag() (in module eval_framework.benchmarks.hellaswag) hellaswag_ellamind() (in module eval_framework.benchmarks.hellaswag_ellamind) hellaswag_olmes() (in module eval_framework.benchmarks.hellaswag) HellaswagReader (class in eval_framework.benchmarks.hellaswag) (class in eval_framework.benchmarks.hellaswag_ellamind) hendrycks_math_ellamind() (in module eval_framework.benchmarks.hendrycks_math_ellamind) hf_revision (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) hf_upload_dir (eval_framework.tasks.eval_config.EvalConfig attribute) hf_upload_repo (eval_framework.tasks.eval_config.EvalConfig attribute) HfDatasetLoader (class in eval_framework.tasks.dataset_loading) HfDatasetRevisions (class in eval_framework.tasks.dataset_revisions) HFUploader (class in eval_framework.result_processors.hf_uploader) higher_is_better (eval_framework.metrics.base.MetricResult attribute) HIGHER_IS_BETTER (eval_framework.metrics.completion.repetition.WordRepetition attribute) higher_is_better (eval_framework.result_processors.base.Result attribute) history (eval_framework.eval_kind.SampleBody attribute) hle_ellamind_bpb_de() (in module eval_framework.benchmarks.hle_ellamind) hle_ellamind_cloze_de() (in module eval_framework.benchmarks.hle_ellamind) hle_ellamind_cloze_native_de() (in module eval_framework.benchmarks.hle_ellamind) hle_ellamind_mc_de() (in module eval_framework.benchmarks.hle_ellamind) hle_ellamind_mc_native_de() (in module eval_framework.benchmarks.hle_ellamind) HleReader (class in eval_framework.benchmarks.hle_ellamind) huber_affine() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) humaneval_bpb() (in module eval_framework.benchmarks.humaneval) humaneval_bpb_olmes_v2() (in module eval_framework.benchmarks.humaneval_ellamind) humaneval_bpb_v2() (in module eval_framework.benchmarks.humaneval) humaneval_context() (in module eval_framework.benchmarks.humaneval) humaneval_de_bpb_olmes() (in module eval_framework.benchmarks.humaneval_ellamind) humaneval_de_instruct() (in module eval_framework.benchmarks.humaneval_ellamind) humaneval_de_olmes() (in module eval_framework.benchmarks.humaneval_ellamind) humaneval_olmes() (in module eval_framework.benchmarks.humaneval) humaneval_olmes_v2() (in module eval_framework.benchmarks.humaneval) (in module eval_framework.benchmarks.humaneval_ellamind) humaneval_plus() (in module eval_framework.benchmarks.humaneval_plus) humaneval_plus_bpb() (in module eval_framework.benchmarks.humaneval_plus) HumanEvalMetricContext (class in eval_framework.benchmarks.humaneval) HumanEvalPlusCodeCompletionAssertion (class in eval_framework.metrics.completion.code_assertion) HUN (eval_framework.tasks.task_style.Language attribute) I id() (eval_framework.composed.ComposedBenchmark method) (eval_framework.contract.Benchmark method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) IdentifierMean (class in eval_framework.metrics.aggregators.aggregators) Identity (class in eval_framework.metrics.aggregators.aggregators) IdkStyle (class in eval_framework.tasks.task_style) ifeval() (in module eval_framework.benchmarks.ifeval) ifeval_de() (in module eval_framework.benchmarks.ifeval) IFEvalMetric (class in eval_framework.metrics.completion.ifeval) IFEvalMetricContext (class in eval_framework.metrics.completion.ifeval) IMAGE (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion attribute) implicit (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) initial_prompt() (eval_framework.tasks.task_style.ClozeStyle method) (eval_framework.tasks.task_style.IdkStyle method) (eval_framework.tasks.task_style.MCStyle method) (eval_framework.tasks.task_style.TaskStyler method) instruct_reconstruct() (in module eval_framework.benchmarks.mbpp) instruction_id_list (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) InstrumentedBitsPerByte (class in eval_framework.metrics.loglikelihood.bpb_variants_instrumented) intercept (eval_framework.metrics.loglikelihood.bpb_variants_estimators.AffineFit attribute) is_column_count_respected (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) is_equiv_hendrycks() (in module eval_framework.metrics.completion.minerva_math_utils) is_equiv_minerva() (in module eval_framework.metrics.completion.minerva_math_utils) is_just_json (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) is_leaf (eval_framework.suite.TaskSuite property) is_registered() (in module eval_framework.tasks.registry) is_retryable() (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.openai.OpenAIModel method) (eval_framework.llm.retry.RetryingLLM method) is_separator_respected (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) is_valid_json (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) ITA (eval_framework.tasks.task_style.Language attribute) items() (eval_framework.tasks.registry.Registry method) iterate_result_candidates() (eval_framework.composed.ComposedEval method) (eval_framework.contract.Eval method) J json_parsing_error (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) JsonFormat (class in eval_framework.metrics.completion.json_format) JsonFormatEvaluation (class in eval_framework.metrics.completion.json_format) judge_model_args (eval_framework.tasks.eval_config.EvalConfig attribute) K key (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) (eval_framework.result_processors.base.Result attribute) KEYS (eval_framework.metrics.base.BaseMetric attribute) (eval_framework.metrics.completion.csv_format.CSVFormat attribute) (eval_framework.metrics.completion.drop_completion.DropF1ExactMatch attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletion attribute) (eval_framework.metrics.efficiency.finish_reason.FinishReason attribute) (eval_framework.metrics.efficiency.token_counters.TokenCounts attribute) L label (eval_framework.subjects.Subject attribute) Language (class in eval_framework.tasks.task_style) language (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale attribute) (eval_framework.benchmarks.utils.ellamind_languages.EllamindLanguage attribute) (eval_framework.metrics.completion.multipl_e_assertion.MultiPLEMetricContext attribute) LanguageChecker (class in eval_framework.metrics.completion.language_checker) LanguageConsistencyChecker (class in eval_framework.metrics.completion.language_checker) LanguageRawConsistencyChecker (class in eval_framework.metrics.completion.language_checker) last_boxed_only_string() (in module eval_framework.metrics.completion.minerva_math_utils) last_match() (in module eval_framework.answer) Lazy (class in eval_framework.tasks.lazy) (class in eval_framework.tasks.registry) lbpp() (in module eval_framework.benchmarks.lbpp) lbpp_bpb() (in module eval_framework.benchmarks.lbpp) LBPPCodeCompletionAssertion (class in eval_framework.metrics.completion.code_assertion) LengthControl (class in eval_framework.metrics.completion.length_control) LengthRequirementType (class in eval_framework.metrics.completion.length_control) LengthRequirementUnit (class in eval_framework.metrics.completion.length_control) list_ground_truth_candidates() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) (in module eval_framework.metrics.loglikelihood.bpb_variants_instrumented) ListOfSubjects (class in eval_framework.subjects) llm_args (eval_framework.tasks.eval_config.EvalConfig attribute) llm_class (eval_framework.tasks.eval_config.EvalConfig attribute) llm_judge() (eval_framework.tasks.eval_config.EvalConfig method) llm_judge_class (eval_framework.tasks.eval_config.EvalConfig attribute) llm_judge_prompt (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) llm_judge_response (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) LLM_NAME (eval_framework.llm.openai.OpenAIModel attribute) llm_name (eval_framework.result_processors.base.Result attribute) LLMUnavailable load() (eval_framework.benchmarks.mmlu_prox.MMLUProXDatasetLoader method) (eval_framework.suite.TaskSuite class method) (eval_framework.tasks.dataset_loading.DatasetLoader method) (eval_framework.tasks.dataset_loading.FixedHfConfigLoader method) (eval_framework.tasks.dataset_loading.HfDatasetLoader method) load_extra_tasks() (in module eval_framework.tasks.task_loader) load_from_py() (eval_framework.suite.TaskSuite class method) load_from_yaml() (eval_framework.suite.TaskSuite class method) load_key (eval_framework.subjects.Subject attribute) load_metadata() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) load_metrics_results() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) load_modules_from_directory() (in module eval_framework.tasks.task_loader) load_responses() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) loader() (eval_framework.benchmarks.mmlu_prox.MMLUProXDatasetPolicy method) (eval_framework.tasks.dataset_loading.DatasetPolicy method) (eval_framework.tasks.dataset_loading.SubjectColumn method) (eval_framework.tasks.dataset_loading.Subset method) (eval_framework.tasks.dataset_revisions.FixedHfConfig method) (eval_framework.tasks.dataset_revisions.Pinned method) LogicError LOGLIKELIHOODS (eval_framework.contract.ResponseType attribute) logprobs() (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.openai.OpenAIEmbeddingModel method) (eval_framework.llm.openai.OpenAIModel method) (eval_framework.llm.retry.RetryingLLM method) ls_bpb() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) M main() (in module eval_framework.main) make_prefix_items() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) marginal_rate_curve() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) markdown_doc() (eval_framework.composed.ComposedBenchmark method) (eval_framework.contract.Benchmark method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) (in module eval_framework.tasks.markdown_doc) math500_v2() (in module eval_framework.benchmarks.math_reasoning) math500_with_bug() (in module eval_framework.benchmarks.math_reasoning) mathminerva_bpb() (in module eval_framework.benchmarks.math_reasoning) mathminerva_olmes() (in module eval_framework.benchmarks.math_reasoning) mathminerva_olmes_nonl() (in module eval_framework.benchmarks.math_reasoning) MathMinervaCompletion (class in eval_framework.metrics.completion.math_minerva_completion) MathMinervaCompletionDE (class in eval_framework.metrics.completion.math_minerva_completion) MathMinervaCompletionFR (class in eval_framework.metrics.completion.math_minerva_completion) MathMinervaCompletionRelaxed (class in eval_framework.metrics.completion.math_minerva_completion) MathMinervaCompletionRelaxedDE (class in eval_framework.metrics.completion.math_minerva_completion) MathMinervaCompletionRelaxedFR (class in eval_framework.metrics.completion.math_minerva_completion) MathReasoningCompletion (class in eval_framework.metrics.completion.math_reasoning_completion) MAX (eval_framework.metrics.completion.length_control.LengthRequirementType attribute) max_backoff_s (eval_framework.llm.retry.RetryPolicy attribute) max_tokens (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) max_tokens() (eval_framework.answer.AnswerPolicy method) (eval_framework.answer.ExtractFromCompletion method) (eval_framework.answer.PickFromCandidates method) (eval_framework.answer.ReconstructProgram method) MAX_WORKERS (eval_framework.metrics.base.BaseMetric attribute) (eval_framework.metrics.llm.base.BaseLLMJudgeMetric attribute) mbpp_bpb() (in module eval_framework.benchmarks.mbpp) mbpp_bpb_evalplus() (in module eval_framework.benchmarks.mbpp) mbpp_context() (in module eval_framework.benchmarks.mbpp) mbpp_ellamind() (in module eval_framework.benchmarks.mbpp_ellamind) mbpp_evalplus() (in module eval_framework.benchmarks.mbpp) mbpp_ground_truth() (in module eval_framework.benchmarks.mbpp) mbpp_olmes() (in module eval_framework.benchmarks.mbpp) MbppLocale (class in eval_framework.benchmarks.mbpp_ellamind) MBPPMetricContext (class in eval_framework.benchmarks.mbpp) mc_preamble() (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale method) mc_template (eval_framework.benchmarks.mmlu_prox.MMLUProXLocale attribute) MCCompletionStyle (class in eval_framework.tasks.task_style) MCStyle (class in eval_framework.tasks.task_style) mean_cost_curve() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) mean_of_ratios() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) messages (eval_framework.contract.ResultCandidate attribute) messages() (eval_framework.benchmarks.cot.Cot method) (eval_framework.benchmarks.winogrande.PartialEval method) (eval_framework.eval_kind.Choice method) (eval_framework.eval_kind.EvalKind method) (eval_framework.eval_kind.Generative method) metadata() (eval_framework.benchmarks.mmlu_prox.MMLUProXDatasetLoader method) (eval_framework.eval_kind.Choice method) (eval_framework.eval_kind.EvalKind method) (eval_framework.fewshot.FewShotGenerator method) (eval_framework.fewshot.FewShotSource method) (eval_framework.fewshot.FewShotSplit method) (eval_framework.fewshot.Predefined method) (eval_framework.fewshot.SampleSplit method) (eval_framework.tasks.dataset_loading.DatasetLoader method) (eval_framework.tasks.dataset_loading.FixedHfConfigLoader method) (eval_framework.tasks.dataset_loading.HfDatasetLoader method) method (eval_framework.metrics.loglikelihood.bpb_variants_estimators.AffineFit attribute) (eval_framework.suite.SuiteAggregate attribute) metric (eval_framework.suite.MetricSource attribute) metric_class_name (eval_framework.result_processors.base.Result attribute) metric_name (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) MetricResult (class in eval_framework.metrics.base) metrics (eval_framework.benchmarks.hendrycks_math_ellamind.MinervaLocale attribute) (eval_framework.tasks.task_style.BPBStyle attribute) (eval_framework.tasks.task_style.ClozeStyle attribute) (eval_framework.tasks.task_style.IdkStyle attribute) (eval_framework.tasks.task_style.MCCompletionStyle attribute) (eval_framework.tasks.task_style.MCStyle attribute) (eval_framework.tasks.task_style.TaskStyler attribute), [1] metrics() (eval_framework.answer.AnswerPolicy method) (eval_framework.answer.ExtractFromCompletion method) (eval_framework.answer.PickFromCandidates method) (eval_framework.answer.ReconstructProgram method) (eval_framework.benchmarks.cot.Cot method) (eval_framework.benchmarks.winogrande.PartialEval method) (eval_framework.composed.ComposedBenchmark method) (eval_framework.contract.Benchmark method) (eval_framework.eval_kind.Choice method) (eval_framework.eval_kind.EvalKind method) (eval_framework.eval_kind.Generative method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) MetricSource (class in eval_framework.suite) MIN (eval_framework.metrics.completion.length_control.LengthRequirementType attribute) MinervaLocale (class in eval_framework.benchmarks.hendrycks_math_ellamind) missing_references() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions method) mmlu() (in module eval_framework.benchmarks.mmlu) mmlu_cot() (in module eval_framework.benchmarks.mmlu) mmlu_full_text() (in module eval_framework.benchmarks.mmlu) mmlu_idk() (in module eval_framework.benchmarks.mmlu) mmlu_olmes() (in module eval_framework.benchmarks.mmlu) mmlu_pro() (in module eval_framework.benchmarks.mmlu_pro) mmlu_pro_cot() (in module eval_framework.benchmarks.mmlu_pro) mmlu_pro_cot_v2() (in module eval_framework.benchmarks.mmlu_pro) mmlu_pro_idk() (in module eval_framework.benchmarks.mmlu_pro) mmlu_pro_olmes() (in module eval_framework.benchmarks.mmlu_pro) mmlu_prox() (in module eval_framework.benchmarks.mmlu_prox) mmlu_prox_lilab_de_cot() (in module eval_framework.benchmarks.mmlu_prox) mmlu_prox_lilab_de_cot_v2() (in module eval_framework.benchmarks.mmlu_prox) mmlu_prox_lite_lilab_de_cot() (in module eval_framework.benchmarks.mmlu_prox) mmlu_prox_lite_lilab_de_cot_v2() (in module eval_framework.benchmarks.mmlu_prox) MmluProReader (class in eval_framework.benchmarks.mmlu_pro) MMLUProXDatasetLoader (class in eval_framework.benchmarks.mmlu_prox) MMLUProXDatasetPolicy (class in eval_framework.benchmarks.mmlu_prox) MMLUProXLocale (class in eval_framework.benchmarks.mmlu_prox) MMLUProXReader (class in eval_framework.benchmarks.mmlu_prox) MmluReader (class in eval_framework.benchmarks.mmlu) model_config (eval_framework.base_config.BaseConfig attribute) (eval_framework.benchmarks.humaneval.HumanEvalMetricContext attribute) (eval_framework.benchmarks.mbpp.MBPPMetricContext attribute) (eval_framework.contract.ResultCandidate attribute) (eval_framework.metrics.base.MetricResult attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext attribute) (eval_framework.metrics.completion.csv_format.CSVFormatEvaluation attribute) (eval_framework.metrics.completion.drop_completion.DropMetricContext attribute) (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) (eval_framework.metrics.completion.multipl_e_assertion.MultiPLEMetricContext attribute) (eval_framework.metrics.completion.placeholder_checker.PlaceholderCheckerMetricContext attribute) (eval_framework.metrics.completion.text_counter.ParagraphCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.SentenceCounterMetricContext attribute) (eval_framework.metrics.completion.text_counter.WordCounterMetricContext attribute) (eval_framework.result_processors.base.Result attribute) (eval_framework.suite.MetricSource attribute) (eval_framework.suite.SuiteAggregate attribute) (eval_framework.suite.SuiteResult attribute) (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) model_json_dump() (eval_framework.tasks.eval_config.EvalConfig method) model_json_robust_subset_dump() (eval_framework.tasks.eval_config.EvalConfig method) module eval_framework eval_framework.answer eval_framework.base_config eval_framework.benchmarks eval_framework.benchmarks.arc eval_framework.benchmarks.arc_de eval_framework.benchmarks.arc_ellamind eval_framework.benchmarks.bigcodebench eval_framework.benchmarks.copa eval_framework.benchmarks.cot eval_framework.benchmarks.csqa eval_framework.benchmarks.csqa_ellamind eval_framework.benchmarks.drop eval_framework.benchmarks.global_mmlu eval_framework.benchmarks.goldenswag eval_framework.benchmarks.gpqa eval_framework.benchmarks.gpqa_ellamind eval_framework.benchmarks.gsm8k eval_framework.benchmarks.gsm8k_ellamind eval_framework.benchmarks.hellaswag eval_framework.benchmarks.hellaswag_ellamind eval_framework.benchmarks.hendrycks_math_ellamind eval_framework.benchmarks.hle_ellamind eval_framework.benchmarks.humaneval eval_framework.benchmarks.humaneval_ellamind eval_framework.benchmarks.humaneval_plus eval_framework.benchmarks.ifeval eval_framework.benchmarks.lbpp eval_framework.benchmarks.math_reasoning eval_framework.benchmarks.mbpp eval_framework.benchmarks.mbpp_ellamind eval_framework.benchmarks.mmlu eval_framework.benchmarks.mmlu_pro eval_framework.benchmarks.mmlu_prox eval_framework.benchmarks.multipl_e eval_framework.benchmarks.naturalqs_open eval_framework.benchmarks.piqa eval_framework.benchmarks.piqa_ellamind eval_framework.benchmarks.sciq eval_framework.benchmarks.simpleqa_ellamind eval_framework.benchmarks.siqa_ellamind eval_framework.benchmarks.social_iqa eval_framework.benchmarks.squad eval_framework.benchmarks.utils eval_framework.benchmarks.utils.ellamind_languages eval_framework.benchmarks.utils.global_mmlu_languages eval_framework.benchmarks.winogrande eval_framework.benchmarks.winogrande_ellamind eval_framework.choices eval_framework.composed eval_framework.contract eval_framework.eval_kind eval_framework.evaluation_generator eval_framework.exceptions eval_framework.fewshot eval_framework.llm eval_framework.llm.base eval_framework.llm.models eval_framework.llm.openai eval_framework.llm.retry eval_framework.logger eval_framework.main eval_framework.metrics eval_framework.metrics.aggregators eval_framework.metrics.aggregators.aggregators eval_framework.metrics.base eval_framework.metrics.completion eval_framework.metrics.completion.accuracy_completion eval_framework.metrics.completion.code_assertion eval_framework.metrics.completion.code_execution_pass_at_one eval_framework.metrics.completion.concordance_index eval_framework.metrics.completion.csv_format eval_framework.metrics.completion.drop_completion eval_framework.metrics.completion.exponential_similarity eval_framework.metrics.completion.f1 eval_framework.metrics.completion.format_checker eval_framework.metrics.completion.grid_difference eval_framework.metrics.completion.ifeval eval_framework.metrics.completion.json_format eval_framework.metrics.completion.language_checker eval_framework.metrics.completion.length_control eval_framework.metrics.completion.math_minerva_completion eval_framework.metrics.completion.math_reasoning_completion eval_framework.metrics.completion.minerva_math_utils eval_framework.metrics.completion.multipl_e_assertion eval_framework.metrics.completion.placeholder_checker eval_framework.metrics.completion.repetition eval_framework.metrics.completion.rouge_1 eval_framework.metrics.completion.rouge_2 eval_framework.metrics.completion.rouge_geometric_mean eval_framework.metrics.completion.rouge_l eval_framework.metrics.completion.text_counter eval_framework.metrics.efficiency eval_framework.metrics.efficiency.bytes_per_sequence_position eval_framework.metrics.efficiency.finish_reason eval_framework.metrics.efficiency.token_counters eval_framework.metrics.llm eval_framework.metrics.llm.base eval_framework.metrics.loglikelihood eval_framework.metrics.loglikelihood.accuracy_loglikelihood eval_framework.metrics.loglikelihood.base eval_framework.metrics.loglikelihood.bits_per_byte eval_framework.metrics.loglikelihood.bpb_variants eval_framework.metrics.loglikelihood.bpb_variants_common eval_framework.metrics.loglikelihood.bpb_variants_estimators eval_framework.metrics.loglikelihood.bpb_variants_instrumented eval_framework.metrics.loglikelihood.bpb_variants_prefix eval_framework.metrics.loglikelihood.confidence_weighted_accuracy eval_framework.metrics.loglikelihood.dcs eval_framework.metrics.loglikelihood.probability_mass eval_framework.metrics.loglikelihood.ternary eval_framework.response_generator eval_framework.result_processors eval_framework.result_processors.base eval_framework.result_processors.hf_uploader eval_framework.result_processors.result_processor eval_framework.result_processors.wandb_uploader eval_framework.run eval_framework.run_direct eval_framework.subjects eval_framework.suite eval_framework.tasks eval_framework.tasks.base eval_framework.tasks.dataset_loading eval_framework.tasks.dataset_revisions eval_framework.tasks.eval_config eval_framework.tasks.lazy eval_framework.tasks.markdown_doc eval_framework.tasks.registry eval_framework.tasks.task_loader eval_framework.tasks.task_names eval_framework.tasks.task_style eval_framework.tasks.utils MULTIPLE_CHOICE (eval_framework.tasks.task_style.TaskStyle attribute) MultiPLECodeAssertion (class in eval_framework.metrics.completion.multipl_e_assertion) MultiPLEMetricContext (class in eval_framework.metrics.completion.multipl_e_assertion) N name (eval_framework.llm.base.BaseLLM property) (eval_framework.llm.retry.RetryingLLM property) (eval_framework.metrics.aggregators.aggregators.Aggregator attribute) NAME (eval_framework.metrics.base.BaseMetric attribute) (eval_framework.metrics.completion.accuracy_completion.AccuracyCompletion attribute) (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOne attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneWithCodebench attribute) (eval_framework.metrics.completion.concordance_index.ConcordanceIndex attribute) (eval_framework.metrics.completion.csv_format.CSVFormat attribute) (eval_framework.metrics.completion.drop_completion.DropF1ExactMatch attribute) (eval_framework.metrics.completion.exponential_similarity.ExponentialSimilarity attribute) (eval_framework.metrics.completion.f1.F1 attribute) (eval_framework.metrics.completion.f1.F1SquadNormalized attribute) (eval_framework.metrics.completion.format_checker.CheckJsonFormat attribute) (eval_framework.metrics.completion.format_checker.CheckPostScriptFormat attribute) (eval_framework.metrics.completion.grid_difference.GridDifference attribute) (eval_framework.metrics.completion.ifeval.IFEvalMetric attribute) (eval_framework.metrics.completion.json_format.JsonFormat attribute) (eval_framework.metrics.completion.language_checker.GermanCompletionChecker attribute) (eval_framework.metrics.completion.language_checker.LanguageChecker attribute) (eval_framework.metrics.completion.language_checker.LanguageConsistencyChecker attribute) (eval_framework.metrics.completion.language_checker.LanguageRawConsistencyChecker attribute) (eval_framework.metrics.completion.length_control.LengthControl attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletion attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionDE attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionFR attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionRelaxed attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionRelaxedDE attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionRelaxedFR attribute) (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) (eval_framework.metrics.completion.multipl_e_assertion.MultiPLECodeAssertion attribute) (eval_framework.metrics.completion.placeholder_checker.PlaceholderChecker attribute) (eval_framework.metrics.completion.repetition.WordRepetition attribute) (eval_framework.metrics.completion.rouge_1.ROUGE_1 attribute) (eval_framework.metrics.completion.rouge_2.ROUGE_2 attribute) (eval_framework.metrics.completion.rouge_geometric_mean.ROUGE_GEOMETRIC_MEAN attribute) (eval_framework.metrics.completion.rouge_l.ROUGE_L attribute) (eval_framework.metrics.completion.text_counter.ParagraphCounter attribute) (eval_framework.metrics.completion.text_counter.ResponseToOriginalLengthRatio attribute) (eval_framework.metrics.completion.text_counter.SentenceCounter attribute) (eval_framework.metrics.completion.text_counter.WordCounter attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesCompletion attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesLoglikelihood attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsCompletion attribute) (eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsLoglikelihood attribute) (eval_framework.metrics.efficiency.finish_reason.FinishReason attribute) (eval_framework.metrics.efficiency.token_counters.TokenCounts attribute) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyBayesianLoglikelihood attribute) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyLoglikelihood attribute) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyNormLoglikelihood attribute) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.PartialEvalAccuracy attribute) (eval_framework.metrics.loglikelihood.bits_per_byte.BitsPerByteLoglikelihood attribute) (eval_framework.metrics.loglikelihood.bpb_variants.BitsPerByteVariantsLoglikelihood attribute) (eval_framework.metrics.loglikelihood.bpb_variants_instrumented.InstrumentedBitsPerByte attribute) (eval_framework.metrics.loglikelihood.bpb_variants_prefix.PrefixBitsPerByte attribute) (eval_framework.metrics.loglikelihood.confidence_weighted_accuracy.ConfidenceWeightedAccuracy attribute) (eval_framework.metrics.loglikelihood.dcs.DistributionalCorrectnessScore attribute) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMass attribute) (eval_framework.metrics.loglikelihood.probability_mass.ProbabilityMassNorm attribute) (eval_framework.metrics.loglikelihood.ternary.TernaryScore attribute) name (eval_framework.suite.SuiteAggregate attribute) (eval_framework.suite.SuiteResult attribute) (eval_framework.suite.TaskSuite attribute) NAMES (eval_framework.metrics.base.BaseMetric attribute) natural_qs_open() (in module eval_framework.benchmarks.naturalqs_open) natural_qs_open_mc_olmes() (in module eval_framework.benchmarks.naturalqs_open) NLD (eval_framework.tasks.task_style.Language attribute) NoContext() (in module eval_framework.eval_kind) NoFewShot (class in eval_framework.fewshot) normalize() (eval_framework.metrics.completion.f1.F1 method) (eval_framework.metrics.completion.f1.F1SquadNormalized method) normalize_expression() (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion method) normalize_final_answer() (in module eval_framework.metrics.completion.minerva_math_utils) normalized_gold_from_solution() (in module eval_framework.metrics.completion.minerva_math_utils) NoSubject (class in eval_framework.subjects) num_fewshot (eval_framework.result_processors.base.Result attribute) (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) num_placeholders (eval_framework.metrics.completion.placeholder_checker.PlaceholderCheckerMetricContext attribute) num_revisions() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions method) num_samples (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) O offset (eval_framework.metrics.loglikelihood.bpb_variants_estimators.PrefixItem attribute) olmes_cue (eval_framework.benchmarks.mbpp_ellamind.MbppLocale attribute) olmes_reconstruct() (in module eval_framework.benchmarks.mbpp) olmes_target() (in module eval_framework.benchmarks.humaneval) ols_affine() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) ols_leverage_weights() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) OpenAIEmbeddingModel (class in eval_framework.llm.openai) OpenAIModel (class in eval_framework.llm.openai) output (eval_framework.tasks.utils.ExecutionResult attribute), [1] output_dir (eval_framework.tasks.eval_config.EvalConfig attribute) output_parse_fn (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext attribute) P package_downloads (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) PACKAGES (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion attribute) (eval_framework.metrics.completion.code_assertion.HumanEvalPlusCodeCompletionAssertion attribute) (eval_framework.metrics.completion.code_assertion.LBPPCodeCompletionAssertion attribute) paragraph_count (eval_framework.metrics.completion.text_counter.ParagraphCounterMetricContext attribute) ParagraphCounter (class in eval_framework.metrics.completion.text_counter) ParagraphCounterMetricContext (class in eval_framework.metrics.completion.text_counter) PARAGRAPHS (eval_framework.metrics.completion.length_control.LengthRequirementUnit attribute) parse_args() (in module eval_framework.run) parse_strings_to_task_or_suite() (in module eval_framework.suite) PartialEval (class in eval_framework.benchmarks.winogrande) PartialEvalAccuracy (class in eval_framework.metrics.loglikelihood.accuracy_loglikelihood) PassAtK (class in eval_framework.metrics.aggregators.aggregators) pause_before() (eval_framework.llm.retry.RetryPolicy method) pbpb() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) per_byte_cost() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) PickFromCandidates (class in eval_framework.answer) Pinned (class in eval_framework.tasks.dataset_revisions) pinned_by_framework() (in module eval_framework.tasks.dataset_revisions) pinned_frozen() (in module eval_framework.tasks.dataset_revisions) pinned_revision() (in module eval_framework.tasks.dataset_revisions) piqa() (in module eval_framework.benchmarks.piqa) piqa_ellamind() (in module eval_framework.benchmarks.piqa_ellamind) piqa_idk() (in module eval_framework.benchmarks.piqa) piqa_olmes() (in module eval_framework.benchmarks.piqa) PiqaReader (class in eval_framework.benchmarks.piqa) (class in eval_framework.benchmarks.piqa_ellamind) PlaceholderChecker (class in eval_framework.metrics.completion.placeholder_checker) PlaceholderCheckerMetricContext (class in eval_framework.metrics.completion.placeholder_checker) POL (eval_framework.tasks.task_style.Language attribute) Pool (class in eval_framework.fewshot) POR (eval_framework.tasks.task_style.Language attribute) possible_completions (eval_framework.contract.ResultCandidate attribute) (eval_framework.eval_kind.SampleBody attribute) post_process_completion() (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.retry.RetryingLLM method) preamble (eval_framework.benchmarks.mbpp_ellamind.MbppLocale attribute) preamble() (eval_framework.benchmarks.utils.global_mmlu_languages.GlobalMMLULocale method) preamble_stop (eval_framework.benchmarks.mbpp_ellamind.MbppLocale attribute) preamble_template (eval_framework.benchmarks.utils.global_mmlu_languages.GlobalMMLULocale attribute) Predefined (class in eval_framework.fewshot) prefix_curve() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) prefix_decompose() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) PrefixBitsPerByte (class in eval_framework.metrics.loglikelihood.bpb_variants_prefix) PrefixItem (class in eval_framework.metrics.loglikelihood.bpb_variants_estimators) prepare() (eval_framework.fewshot.FewShotGenerator method) (eval_framework.metrics.base.BaseMetric method) (eval_framework.metrics.loglikelihood.accuracy_loglikelihood.AccuracyBayesianLoglikelihood method) prior_bpb() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) ProbabilityMass (class in eval_framework.metrics.loglikelihood.probability_mass) ProbabilityMassNorm (class in eval_framework.metrics.loglikelihood.probability_mass) problem_prefix (eval_framework.benchmarks.hendrycks_math_ellamind.MinervaLocale attribute) prompt (eval_framework.benchmarks.humaneval.HumanEvalMetricContext attribute) (eval_framework.eval_kind.SampleBody attribute) (eval_framework.fewshot.FewshotExample attribute) (eval_framework.metrics.completion.ifeval.IFEvalMetricContext attribute) (eval_framework.metrics.completion.multipl_e_assertion.MultiPLEMetricContext attribute) Q question (eval_framework.benchmarks.humaneval.SingleGoldReader attribute) (eval_framework.benchmarks.mbpp.SingleGoldReader attribute) question_cue_text() (in module eval_framework.tasks.task_style) question_label (eval_framework.benchmarks.utils.global_mmlu_languages.GlobalMMLULocale attribute) question_prefix (eval_framework.tasks.task_style.TaskStyler attribute), [1] R r2 (eval_framework.metrics.loglikelihood.bpb_variants_estimators.AffineFit attribute) randomize_judge_order (eval_framework.tasks.eval_config.EvalConfig attribute) raw_question (eval_framework.choices.ChoiceFields attribute) read() (eval_framework.benchmarks.arc.ArcReader method) (eval_framework.benchmarks.arc_de.ArcDeReader method) (eval_framework.benchmarks.arc_ellamind.ArcEllamindReader method) (eval_framework.benchmarks.copa.CopaReader method) (eval_framework.benchmarks.csqa.CommonsenseQaReader method) (eval_framework.benchmarks.csqa_ellamind.CsqaReader method) (eval_framework.benchmarks.gpqa.GpqaReader method) (eval_framework.benchmarks.gpqa_ellamind.GpqaReader method) (eval_framework.benchmarks.hellaswag.HellaswagReader method) (eval_framework.benchmarks.hellaswag_ellamind.HellaswagReader method) (eval_framework.benchmarks.hle_ellamind.HleReader method) (eval_framework.benchmarks.humaneval.SingleGoldReader method) (eval_framework.benchmarks.mbpp.SingleGoldReader method) (eval_framework.benchmarks.mmlu.MmluReader method) (eval_framework.benchmarks.mmlu_pro.MmluProReader method) (eval_framework.benchmarks.mmlu_prox.MMLUProXReader method) (eval_framework.benchmarks.piqa.PiqaReader method) (eval_framework.benchmarks.piqa_ellamind.PiqaReader method) (eval_framework.benchmarks.sciq.SciqReader method) (eval_framework.benchmarks.simpleqa_ellamind.SimpleqaReader method) (eval_framework.benchmarks.siqa_ellamind.SiqaReader method) (eval_framework.benchmarks.social_iqa.SocialIqaReader method) (eval_framework.benchmarks.winogrande.WinograndeReader method) (eval_framework.choices.ChoiceReader method) RealtimeCodeExectionContext (class in eval_framework.metrics.completion.code_execution_pass_at_one) ReconstructProgram (class in eval_framework.answer) reference_survival() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) register_all_tasks() (in module eval_framework.tasks.task_names) register_arc_de_tasks() (in module eval_framework.tasks.task_names) register_arc_ellamind_tasks() (in module eval_framework.tasks.task_names) register_arc_tasks() (in module eval_framework.tasks.task_names) register_artifact_upload_function() (in module eval_framework.result_processors.wandb_uploader) register_bigcodebench_tasks() (in module eval_framework.tasks.task_names) register_copa_tasks() (in module eval_framework.tasks.task_names) register_csqa_ellamind_tasks() (in module eval_framework.tasks.task_names) register_csqa_tasks() (in module eval_framework.tasks.task_names) register_drop_tasks() (in module eval_framework.tasks.task_names) register_global_mmlu_tasks() (in module eval_framework.tasks.task_names) register_goldenswag_tasks() (in module eval_framework.tasks.task_names) register_gpqa_ellamind_tasks() (in module eval_framework.tasks.task_names) register_gpqa_tasks() (in module eval_framework.tasks.task_names) register_gsm8k_ellamind_tasks() (in module eval_framework.tasks.task_names) register_gsm8k_tasks() (in module eval_framework.tasks.task_names) register_hellaswag_ellamind_tasks() (in module eval_framework.tasks.task_names) register_hellaswag_tasks() (in module eval_framework.tasks.task_names) register_hendrycks_math_ellamind_tasks() (in module eval_framework.tasks.task_names) register_hle_ellamind_tasks() (in module eval_framework.tasks.task_names) register_humaneval_ellamind_tasks() (in module eval_framework.tasks.task_names) register_humaneval_plus_tasks() (in module eval_framework.tasks.task_names) register_humaneval_tasks() (in module eval_framework.tasks.task_names) register_ifeval_tasks() (in module eval_framework.tasks.task_names) register_lbpp_tasks() (in module eval_framework.tasks.task_names) register_math_reasoning_tasks() (in module eval_framework.tasks.task_names) register_mbpp_ellamind_tasks() (in module eval_framework.tasks.task_names) register_mbpp_tasks() (in module eval_framework.tasks.task_names) register_mmlu_pro_tasks() (in module eval_framework.tasks.task_names) register_mmlu_prox_tasks() (in module eval_framework.tasks.task_names) register_mmlu_tasks() (in module eval_framework.tasks.task_names) register_multipl_e_tasks() (in module eval_framework.tasks.task_names) register_naturalqs_open_tasks() (in module eval_framework.tasks.task_names) register_piqa_ellamind_tasks() (in module eval_framework.tasks.task_names) register_piqa_tasks() (in module eval_framework.tasks.task_names) register_sciq_tasks() (in module eval_framework.tasks.task_names) register_simpleqa_ellamind_tasks() (in module eval_framework.tasks.task_names) register_siqa_ellamind_tasks() (in module eval_framework.tasks.task_names) register_social_iqa_tasks() (in module eval_framework.tasks.task_names) register_squad_tasks() (in module eval_framework.tasks.task_names) register_tasks_from_module() (in module eval_framework.tasks.task_loader) register_tasks_from_modules() (in module eval_framework.tasks.task_loader) register_winogrande_ellamind_tasks() (in module eval_framework.tasks.task_names) register_winogrande_tasks() (in module eval_framework.tasks.task_names) registered_task_names() (in module eval_framework.tasks.registry) Registry (class in eval_framework.tasks.registry) RELAXED (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletion attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionRelaxed attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionRelaxedDE attribute) (eval_framework.metrics.completion.math_minerva_completion.MathMinervaCompletionRelaxedFR attribute) remove_boxed() (in module eval_framework.metrics.completion.minerva_math_utils) remove_comments() (in module eval_framework.metrics.completion.json_format) REMOVED_EXPRESSIONS_FORMAT (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) REMOVED_EXPRESSIONS_UNITS (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) render() (eval_framework.fewshot.ChoiceRenderer method) (eval_framework.fewshot.FewShotRenderer method) (eval_framework.fewshot.FunctionRenderer method) repeats (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) resolve_count() (eval_framework.fewshot.FewShotSource method) (eval_framework.fewshot.FewShotSplit method) (eval_framework.fewshot.Predefined method) (eval_framework.fewshot.SampleSplit method) resolve_nstar() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) resolve_overwrite_subjects() (in module eval_framework.tasks.base) resolve_to_evalconfig_kwargs() (in module eval_framework.suite) response_type (eval_framework.tasks.task_style.ClozeStyle attribute) (eval_framework.tasks.task_style.IdkStyle attribute) (eval_framework.tasks.task_style.MCCompletionStyle attribute) (eval_framework.tasks.task_style.MCStyle attribute) (eval_framework.tasks.task_style.TaskStyler attribute), [1] response_type() (eval_framework.answer.AnswerPolicy method) (eval_framework.answer.ExtractFromCompletion method) (eval_framework.answer.PickFromCandidates method) (eval_framework.answer.ReconstructProgram method) (eval_framework.composed.ComposedBenchmark method) (eval_framework.contract.Benchmark method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) ResponseGenerator (class in eval_framework.response_generator) ResponseToOriginalLengthRatio (class in eval_framework.metrics.completion.text_counter) ResponseType (class in eval_framework.contract) Result (class in eval_framework.result_processors.base) result_id (eval_framework.contract.ResultCandidate attribute) (eval_framework.result_processors.base.Result attribute) ResultCandidate (class in eval_framework.contract) ResultProcessor (class in eval_framework.result_processors.base) ResultsFileProcessor (class in eval_framework.result_processors.result_processor) ResultsUploader (class in eval_framework.result_processors.base) RetryingLLM (class in eval_framework.llm.retry) RetryPolicy (class in eval_framework.llm.retry) revision_for() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions method) ROUGE_1 (class in eval_framework.metrics.completion.rouge_1) ROUGE_2 (class in eval_framework.metrics.completion.rouge_2) ROUGE_GEOMETRIC_MEAN (class in eval_framework.metrics.completion.rouge_geometric_mean) ROUGE_L (class in eval_framework.metrics.completion.rouge_l) run() (in module eval_framework.run) run_env (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) run_eval() (eval_framework.evaluation_generator.EvaluationGenerator method) run_python_code() (in module eval_framework.tasks.utils) run_suite() (in module eval_framework.suite) run_with_kwargs() (in module eval_framework.run) RUNTIME_CONFIGS (eval_framework.metrics.completion.code_assertion.CodeCompletionAssertion attribute) RUS (eval_framework.tasks.task_style.Language attribute) S sample_id (eval_framework.contract.ResultCandidate attribute) (eval_framework.result_processors.base.Result attribute) SampleBody (class in eval_framework.eval_kind) samples() (eval_framework.benchmarks.cot.Cot method) (eval_framework.benchmarks.winogrande.PartialEval method) (eval_framework.eval_kind.Choice method) (eval_framework.eval_kind.EvalKind method) (eval_framework.eval_kind.Generative method) SampleSplit (class in eval_framework.fewshot) save() (eval_framework.base_config.BaseConfig method) save_aggregated_results() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_intermediate_results (eval_framework.tasks.eval_config.EvalConfig attribute) save_logs (eval_framework.tasks.eval_config.EvalConfig attribute) save_metadata() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_metrics_result() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_metrics_results() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_response() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_responses() (eval_framework.result_processors.base.ResultProcessor method) (eval_framework.result_processors.result_processor.ResultsFileProcessor method) save_suite_results() (in module eval_framework.suite) schema_validation_error (eval_framework.metrics.completion.json_format.JsonFormatEvaluation attribute) sciq_mc_olmes() (in module eval_framework.benchmarks.sciq) SciqReader (class in eval_framework.benchmarks.sciq) select() (eval_framework.subjects.ListOfSubjects method) (eval_framework.subjects.NoSubject method) (eval_framework.subjects.SubjectsSelector method) select_ground_truth() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) (in module eval_framework.metrics.loglikelihood.bpb_variants_instrumented) sentence_count (eval_framework.metrics.completion.text_counter.SentenceCounterMetricContext attribute) SentenceCounter (class in eval_framework.metrics.completion.text_counter) SentenceCounterMetricContext (class in eval_framework.metrics.completion.text_counter) SENTENCES (eval_framework.metrics.completion.length_control.LengthRequirementUnit attribute) SequencePositionsCompletion (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) SequencePositionsLoglikelihood (class in eval_framework.metrics.efficiency.bytes_per_sequence_position) SerializationError serialize_judge_model_args() (eval_framework.tasks.eval_config.EvalConfig method) serialize_llm_class() (eval_framework.tasks.eval_config.EvalConfig method) serialize_llm_judge_class() (eval_framework.tasks.eval_config.EvalConfig method) serialize_output_dir() (eval_framework.tasks.eval_config.EvalConfig method) shuffle_correct_with_distractors() (in module eval_framework.tasks.task_style) simpleqa_ellamind_bpb_de() (in module eval_framework.benchmarks.simpleqa_ellamind) simpleqa_ellamind_cloze_easy_de() (in module eval_framework.benchmarks.simpleqa_ellamind) simpleqa_ellamind_cloze_hard_de() (in module eval_framework.benchmarks.simpleqa_ellamind) simpleqa_ellamind_mc_easy_de() (in module eval_framework.benchmarks.simpleqa_ellamind) simpleqa_ellamind_mc_hard_de() (in module eval_framework.benchmarks.simpleqa_ellamind) SimpleqaReader (class in eval_framework.benchmarks.simpleqa_ellamind) SingleGoldReader (class in eval_framework.benchmarks.humaneval) (class in eval_framework.benchmarks.mbpp) siqa_ellamind_bpb_de() (in module eval_framework.benchmarks.siqa_ellamind) siqa_ellamind_cloze_easy_de() (in module eval_framework.benchmarks.siqa_ellamind) siqa_ellamind_cloze_hard_de() (in module eval_framework.benchmarks.siqa_ellamind) siqa_ellamind_mc_easy_de() (in module eval_framework.benchmarks.siqa_ellamind) siqa_ellamind_mc_hard_de() (in module eval_framework.benchmarks.siqa_ellamind) SiqaReader (class in eval_framework.benchmarks.siqa_ellamind) slope (eval_framework.metrics.loglikelihood.bpb_variants_estimators.AffineFit attribute) snippet_merge_fn (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionPassAtOneContext attribute) (eval_framework.metrics.completion.code_execution_pass_at_one.RealtimeCodeExectionContext attribute) social_iqa_mc_olmes() (in module eval_framework.benchmarks.social_iqa) SocialIqaReader (class in eval_framework.benchmarks.social_iqa) solution_cue (eval_framework.benchmarks.hendrycks_math_ellamind.MinervaLocale attribute) sources (eval_framework.suite.SuiteAggregate attribute) SPA (eval_framework.tasks.task_style.Language attribute) space_stripped_corpus_bpb() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) split (eval_framework.fewshot.FewShotDoc attribute) squad2_ma() (in module eval_framework.benchmarks.squad) squad2_ma_no_sysprompt() (in module eval_framework.benchmarks.squad) squad_olmes() (in module eval_framework.benchmarks.squad) standard_bpb_error() (in module eval_framework.metrics.loglikelihood.bpb_variants_common) (in module eval_framework.metrics.loglikelihood.bpb_variants_instrumented) stop_sequences() (eval_framework.answer.AnswerPolicy method) (eval_framework.answer.ExtractFromCompletion method) (eval_framework.answer.PickFromCandidates method) (eval_framework.answer.ReconstructProgram method) strip_string_hendrycks() (in module eval_framework.metrics.completion.minerva_math_utils) Subject (class in eval_framework.subjects) subject (eval_framework.contract.ResultCandidate attribute) (eval_framework.result_processors.base.Result attribute) subject_encoded_in_column() (eval_framework.tasks.dataset_loading.DatasetPolicy method) subject_names (eval_framework.benchmarks.utils.global_mmlu_languages.GlobalMMLULocale attribute) SubjectColumn (class in eval_framework.tasks.dataset_loading) subjects() (eval_framework.composed.ComposedBenchmark method) (eval_framework.contract.Benchmark method) (eval_framework.tasks.lazy.Lazy method) (eval_framework.tasks.registry.Benchmark method) (eval_framework.tasks.registry.Lazy method) SubjectsSelector (class in eval_framework.subjects) Subset (class in eval_framework.tasks.dataset_loading) subset() (eval_framework.tasks.dataset_loading.DatasetPolicy method) SUBSTITUTIONS (eval_framework.metrics.completion.math_reasoning_completion.MathReasoningCompletion attribute) success (eval_framework.tasks.utils.ExecutionResult attribute), [1] SuiteAggregate (class in eval_framework.suite) SuiteResult (class in eval_framework.suite) summarize_all() (in module eval_framework.metrics.loglikelihood.bpb_variants_estimators) system_prompt (eval_framework.eval_kind.SampleBody attribute) T TARGET (eval_framework.metrics.completion.length_control.LengthRequirementType attribute) task_name (eval_framework.result_processors.base.Result attribute) (eval_framework.suite.TaskSuite property) (eval_framework.tasks.eval_config.EvalConfig attribute) task_names() (eval_framework.tasks.registry.Registry method) task_results (eval_framework.suite.SuiteResult attribute) task_style (eval_framework.tasks.task_style.BPBStyle attribute) (eval_framework.tasks.task_style.ClozeStyle attribute) (eval_framework.tasks.task_style.MCCompletionStyle attribute) (eval_framework.tasks.task_style.MCStyle attribute) (eval_framework.tasks.task_style.TaskStyler attribute), [1] task_subjects (eval_framework.suite.TaskSuite attribute) (eval_framework.tasks.eval_config.EvalConfig attribute) tasks (eval_framework.suite.TaskSuite attribute) TaskStyle (class in eval_framework.tasks.task_style) TaskStyler (class in eval_framework.tasks.task_style) TaskSuite (class in eval_framework.suite) temperature (eval_framework.suite.TaskSuite attribute) TernaryScore (class in eval_framework.metrics.loglikelihood.ternary) test (eval_framework.benchmarks.humaneval.HumanEvalMetricContext attribute) test_code (eval_framework.metrics.completion.code_execution_pass_at_one.CodeExecutionBaseContext attribute) tests (eval_framework.metrics.completion.multipl_e_assertion.MultiPLEMetricContext attribute) tests_code (eval_framework.benchmarks.mbpp.MBPPMetricContext attribute) timeout_handler() (in module eval_framework.metrics.completion.math_reasoning_completion) to_dict() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions method) to_file() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions method) TokenCounts (class in eval_framework.metrics.efficiency.token_counters) tokenize() (eval_framework.metrics.completion.f1.F1 method) top_k (eval_framework.suite.TaskSuite attribute) top_p (eval_framework.suite.TaskSuite attribute) total_bytes (eval_framework.metrics.loglikelihood.bpb_variants_estimators.PrefixItem property) tulu3_cot_prompt() (in module eval_framework.benchmarks.cot) tulu3_cot_prompt_de() (in module eval_framework.benchmarks.gpqa_ellamind) tulu_answer() (in module eval_framework.benchmarks.cot) tulu_answer_de() (in module eval_framework.benchmarks.gpqa_ellamind) tulu_answer_v2() (in module eval_framework.benchmarks.cot) U UKR (eval_framework.tasks.task_style.Language attribute) unittest_merge_snippets() (in module eval_framework.tasks.utils) update_to_latest() (eval_framework.tasks.dataset_revisions.HfDatasetRevisions method) upload() (eval_framework.result_processors.base.ResultsUploader method) (eval_framework.result_processors.hf_uploader.HFUploader method) (eval_framework.result_processors.wandb_uploader.WandbUploader method) V v2_bpb() (in module eval_framework.benchmarks.humaneval) v2_execution() (in module eval_framework.benchmarks.humaneval) v2_prompt() (in module eval_framework.benchmarks.humaneval) validate_judge_model_args() (eval_framework.tasks.eval_config.EvalConfig class method) validate_llm_args() (eval_framework.tasks.eval_config.EvalConfig class method) validate_method() (eval_framework.suite.SuiteAggregate class method) validate_output_dir() (eval_framework.tasks.eval_config.EvalConfig class method) validate_suite() (eval_framework.suite.TaskSuite method) validate_task_name() (in module eval_framework.tasks.registry) value (eval_framework.metrics.base.MetricResult attribute) (eval_framework.result_processors.base.Result attribute) W wandb_entity (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_group (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_project (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_run_id (eval_framework.tasks.eval_config.EvalConfig attribute) wandb_upload_results (eval_framework.tasks.eval_config.EvalConfig attribute) WandbUploader (class in eval_framework.result_processors.wandb_uploader) winogrande_cloze() (in module eval_framework.benchmarks.winogrande) winogrande_ellamind_cloze_de() (in module eval_framework.benchmarks.winogrande_ellamind) winogrande_ellamind_mc_de() (in module eval_framework.benchmarks.winogrande_ellamind) winogrande_ellamind_partial_eval_de() (in module eval_framework.benchmarks.winogrande_ellamind) WinograndeReader (class in eval_framework.benchmarks.winogrande) with_abstention_option() (eval_framework.tasks.task_style.TaskStyler method) with_hf_config() (eval_framework.tasks.dataset_revisions.Pinned method) with_registry() (in module eval_framework.tasks.registry) with_retry() (eval_framework.llm.base.BaseLLM method) (eval_framework.llm.retry.RetryingLLM method) word_count (eval_framework.metrics.completion.text_counter.WordCounterMetricContext attribute) WordCounter (class in eval_framework.metrics.completion.text_counter) WordCounterMetricContext (class in eval_framework.metrics.completion.text_counter) WordRepetition (class in eval_framework.metrics.completion.repetition) WORDS (eval_framework.metrics.completion.length_control.LengthRequirementUnit attribute)