Source code for eval_framework.main
import logging
import os
import shutil
from typing import Literal
import wandb
from eval_framework.evaluation_generator import EvaluationGenerator
from eval_framework.llm.base import BaseLLM
from eval_framework.response_generator import ResponseGenerator
from eval_framework.result_processors.base import Result
from eval_framework.result_processors.hf_uploader import HFUploader
from eval_framework.result_processors.result_processor import ResultsFileProcessor, generate_output_dir
from eval_framework.result_processors.wandb_uploader import WandbUploader
from eval_framework.tasks.eval_config import EvalConfig
from eval_framework.utils.constants import RED, RESET
from eval_framework.utils.logging import setup_logging
logger = logging.getLogger(__name__)
[docs]
def main(
llm: BaseLLM,
config: EvalConfig,
verbosity: int = 1,
) -> list[Result]:
"""Runs the entire evaluation process: responses generation and evaluation."""
# Set up centralized logging early
output_dir = generate_output_dir(llm.name, config)
setup_logging(output_dir=output_dir, log_level=verbosity, log_filename="evaluation.log")
logger.info(f"Output directory for evaluation: {output_dir}")
logger.info(f"{RED}[ Running full evaluation process ------- ]{RESET}")
logger.info(f"Evaluating {llm.name} on {config.task_name}")
logger.info(f"Configuration: num_fewshot={config.num_fewshot}, num_samples={config.num_samples}")
logger.info(f"Output directory: {output_dir}")
file_processor = ResultsFileProcessor(output_dir)
response_generator = ResponseGenerator(llm, config, file_processor)
with wandb.init(
entity=config.wandb_entity,
project=config.wandb_project,
group=(config.wandb_group or llm.name)[:127],
job_type=config.task_name[:63],
id=config.wandb_run_id, # none starts a new run
config=response_generator._get_metadata(),
resume="allow",
mode=_wandb_mode(config.wandb_project),
settings=wandb.Settings(disable_code=True), # ("wandb-history" artifacts not needed)
):
for additional_artifact in os.getenv("WANDB_ADDITIONAL_ARTIFACT_REFERENCES", "").split(","):
if additional_artifact.strip():
wandb.use_artifact(additional_artifact.strip())
response_generator.generate()
evaluator = EvaluationGenerator(config, file_processor)
results = evaluator.run_eval()
upload_success = False
for uploader in [HFUploader(config), WandbUploader(config)]:
upload_success |= uploader.upload(llm.name, config, output_dir)
if config.delete_output_dir_after_upload and upload_success:
logger.warning(f"Deleting output directory '{output_dir}' after successful upload(s)!")
shutil.rmtree(output_dir, ignore_errors=True)
if output_dir.exists() and any(output_dir.iterdir()):
logger.warning("Could not delete output directory, some files remain.")
return results
def _wandb_mode(project: str | None) -> Literal["online", "disabled"] | None:
"""
Checks to see if a WandB API key is found. If not, wandb starts in offline mode.
"""
if project is None:
logger.warning("No WandB project specified, disabling logging.")
return "disabled"
else:
try:
api_key = wandb.api.api_key
if api_key is None:
logger.warning(
"""No wandb API key found. Disabling Wandb logging.
If you have a WandB account set the environment variable 'WANDB_API_KEY'"""
)
return "disabled"
else:
logger.info("Wandb login detected. Using online mode.")
except Exception as e:
logger.warning(f"Wandb login check failed: {e}. Disabling Wandb logging.")
return "disabled"
return "online"