eval_framework

• eval_framework package
  • Subpackages
    • eval_framework.context package
      • Submodules
      • eval_framework.context.determined module
      • eval_framework.context.eval module
      • eval_framework.context.local module
      • Module contents
    • eval_framework.llm package
      • Submodules
      • eval_framework.llm.aleph_alpha module
      • eval_framework.llm.base module
      • eval_framework.llm.huggingface module
      • eval_framework.llm.mistral module
      • eval_framework.llm.models module
      • eval_framework.llm.openai module
      • eval_framework.llm.vllm module
      • Module contents
    • eval_framework.metrics package
      • Subpackages
      • Submodules
      • eval_framework.metrics.base module
      • Module contents
    • eval_framework.result_processors package
      • Submodules
      • eval_framework.result_processors.base module
      • eval_framework.result_processors.hf_uploader module
      • eval_framework.result_processors.result_processor module
      • eval_framework.result_processors.wandb_uploader module
      • Module contents
    • eval_framework.tasks package
      • Subpackages
      • Submodules
      • eval_framework.tasks.base module
      • eval_framework.tasks.eval_config module
      • eval_framework.tasks.perturbation module
      • eval_framework.tasks.registry module
      • eval_framework.tasks.task_loader module
      • eval_framework.tasks.task_names module
      • eval_framework.tasks.task_style module
      • eval_framework.tasks.utils module
      • Module contents
  • Submodules
  • eval_framework.base_config module
    • BaseConfig
      • BaseConfig.as_dict()
      • BaseConfig.from_yaml()
      • BaseConfig.model_config
      • BaseConfig.save()
  • eval_framework.evaluation_generator module
    • EvaluationGenerator
      • EvaluationGenerator.run_eval()
  • eval_framework.exceptions module
    • LogicError
  • eval_framework.logger module
  • eval_framework.main module
    • main()
  • eval_framework.response_generator module
    • ResponseGenerator
      • ResponseGenerator.generate()
    • map_language_to_value()
    • repeat_samples()
  • eval_framework.run module
    • parse_args()
    • run()
    • run_with_kwargs()
  • eval_framework.run_direct module
  • eval_framework.suite module
    • MetricSource
      • MetricSource.child
      • MetricSource.metric
      • MetricSource.model_config
    • SuiteAggregate
      • SuiteAggregate.method
      • SuiteAggregate.model_config
      • SuiteAggregate.name
      • SuiteAggregate.sources
      • SuiteAggregate.validate_method()
    • SuiteResult
      • SuiteResult.aggregates
      • SuiteResult.model_config
      • SuiteResult.name
      • SuiteResult.task_results
    • TaskSuite
      • TaskSuite.aggregates
      • TaskSuite.batch_size
      • TaskSuite.extra_llm_args
      • TaskSuite.get_hyperparam_overrides()
      • TaskSuite.hf_revision
      • TaskSuite.is_leaf
      • TaskSuite.load()
      • TaskSuite.load_from_py()
      • TaskSuite.load_from_yaml()
      • TaskSuite.max_tokens
      • TaskSuite.model_config
      • TaskSuite.name
      • TaskSuite.num_fewshot
      • TaskSuite.num_samples
      • TaskSuite.repeats
      • TaskSuite.task_name
      • TaskSuite.task_subjects
      • TaskSuite.tasks
      • TaskSuite.temperature
      • TaskSuite.top_k
      • TaskSuite.top_p
      • TaskSuite.validate_suite()
    • compute_aggregates()
    • parse_strings_to_task_or_suite()
    • resolve_to_evalconfig_kwargs()
    • run_suite()
    • save_suite_results()
  • Module contents