Source code for eval_framework.response_generator

import logging
import time
from collections.abc import Callable
from datetime import UTC, datetime
from functools import partial
from typing import Any

from tqdm import tqdm

from eval_framework import __version__ as eval_framework_version
from eval_framework.contract import Benchmark, ResponseType, ResultCandidate
from eval_framework.llm.base import BaseLLM
from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
from eval_framework.result_processors.result_processor import ResultsFileProcessor
from eval_framework.shared.types import (
    Completion,
    Loglikelihood,
    RawLoglikelihood,
)
from eval_framework.tasks.base import RANDOM_SEED
from eval_framework.tasks.eval_config import EvalConfig
from eval_framework.tasks.registry import Registry, registry
from eval_framework.utils.constants import RED, RESET
from eval_framework.utils.tqdm_handler import get_disable_bar_flag

logger = logging.getLogger(__name__)


def _require_judge_when_task_uses_one(benchmark: Benchmark, config: EvalConfig) -> None:
    """A task scored by an LLM-judge metric needs a judge model; fail early if the config lacks one."""
    if config.llm_judge_class is not None:
        return
    if any(issubclass(metric, BaseLLMJudgeMetric) for metric in benchmark.metrics()):
        raise ValueError(
            f"Task '{config.task_name}' is scored with an LLM-judge metric, so the eval config must set "
            "llm_judge_class."
        )


[docs] class ResponseGenerator: def __init__( self, llm: BaseLLM, config: EvalConfig, result_processor: ResultsFileProcessor, *, benchmark_registry: Registry | None = None, ) -> None: self.task_name = config.task_name self.llm = llm self.config = config self.result_processor = result_processor self.num_samples = config.num_samples self.save_intermediate_results = config.save_intermediate_results benchmark = (benchmark_registry if benchmark_registry is not None else registry())[config.task_name] _require_judge_when_task_uses_one(benchmark, config) self.task = benchmark.create( config.num_fewshot, config.task_subjects, config.hf_revision, seed=RANDOM_SEED, ) self.response_type = self.task.get_response_type() def _llm_task_param_precedence(self) -> tuple[list[str] | None, int | None]: llm_stop_sequences = getattr(self.llm, "stop_sequences", None) llm_max_tokens = getattr(self.llm, "max_tokens", None) task_stop_sequences = self.task.get_stop_sequences() task_max_tokens = self.config.max_tokens or self.task.get_max_tokens() # if both task and model define a max_token, the smaller value is used max_tokens = min( [x for x in [llm_max_tokens, task_max_tokens] if x is not None], default=None, ) logger.info(f"Set max_tokens to {max_tokens}") # if both task and model define stop sequences, those are merged into one list stop_sequences_merged = (llm_stop_sequences or []) + task_stop_sequences stop_sequences = sorted(list(set(stop_sequences_merged))) if stop_sequences_merged else None logger.info(f"Set stop_sequences to {stop_sequences}") return stop_sequences, max_tokens def _generate_loglikelihoods(self, samples: list[ResultCandidate]) -> list[Loglikelihood]: raw_loglikelihoods: list[RawLoglikelihood] raw_loglikelihoods = self.llm.logprobs(samples) loglikelihood_list = [] for idx, sample in enumerate(samples): raw_loglikelihood = raw_loglikelihoods[idx] assert sample.ground_truth is not None loglikelihood_list.append( Loglikelihood( sample_id=sample.sample_id, result_id=sample.result_id, subject=sample.subject, ground_truth=sample.ground_truth, prompt=raw_loglikelihood.prompt, prompt_num_tokens=raw_loglikelihood.prompt_num_tokens, concat_compression=raw_loglikelihood.concat_compression, loglikelihoods=raw_loglikelihood.loglikelihoods, loglikelihoods_num_tokens=raw_loglikelihood.loglikelihoods_num_tokens, loglikelihoods_per_token=raw_loglikelihood.loglikelihoods_per_token, context=sample.context, ) ) return loglikelihood_list def _generative_output_type_selector( self, ) -> Callable[[list[ResultCandidate]], list[Completion] | list[Loglikelihood]]: match self.response_type: case ResponseType.COMPLETION: stop_sequences, max_tokens = self._llm_task_param_precedence() return partial( self.task.generate_completions, self.llm, stop_sequences=stop_sequences, max_tokens=max_tokens, ) case ResponseType.LOGLIKELIHOODS: return self._generate_loglikelihoods case _: raise KeyError(f"Task type {self.task} not supported") def _run_task_against_model(self) -> list[Completion | Loglikelihood]: logger.info(f"{RED}[ Running task {self.task.display_name()} against model ------------ ]{RESET}") self.start_time, monotonic_start = time.time(), time.monotonic() run_fn = self._generative_output_type_selector() self._verify_loaded_metadata_compatibility() responses = self.result_processor.load_responses() # load responses if present subject_response_id_mapping = self._map_subject_response_ids(responses) self.result_processor.save_metadata(self._get_metadata()) responses = self._curate_responses(responses, subject_response_id_mapping, run_fn) self.end_time, monotonic_end = time.time(), time.monotonic() self.total_time = monotonic_end - monotonic_start self.result_processor.save_metadata(self._get_metadata()) # overwrite with updated timing return responses def _map_subject_response_ids(self, responses: list[Completion | Loglikelihood]) -> dict[str, set[int]]: subject_response_id_mapping = {} if responses: response_subjects = {resp.subject for resp in responses} subject_response_id_mapping = { response_subject: set([resp.result_id for resp in responses if resp.subject == response_subject]) for response_subject in response_subjects } return subject_response_id_mapping def _curate_responses( self, responses: list[Completion | Loglikelihood], subject_response_id_mapping: dict[str, set[int]], generative_output_function: Callable[[list[ResultCandidate]], list[Completion] | list[Loglikelihood]], ) -> list[Completion | Loglikelihood]: def _process_batch(samples_batch: list[ResultCandidate]) -> None: if not samples_batch: return if len(samples_batch) > 1: logger.info("Processing batch...") responses_batch = generative_output_function(samples_batch) responses.extend(responses_batch) if self.save_intermediate_results: for response in responses_batch: self.result_processor.save_response(response) # In order to enable parallelism we group samples in batches and send them in parallel to the `run_fn`. # The BaseLLM class is then in charge of managing the parallelism (eg, using AsyncClient in API models). # If samples_batch_size = 1, samples are run sequentially; in any case, we return here after finishing each # individual batch to save cached results. samples_batch_size = self.config.batch_size repeats = self.config.repeats # Calculate total samples for progress bar - use num_samples or iterate to count if self.num_samples is None: # Count samples by iterating (this might be expensive for large datasets) total_num_samples = sum(1 for _ in self.task.iterate_result_candidates(None, repeats=repeats)) else: total_num_samples = self.num_samples * repeats samples_batch: list[ResultCandidate] = [] with tqdm( total=total_num_samples, desc=f"Processing {self.response_type.value}", disable=get_disable_bar_flag(), ) as pbar: for i, sample in enumerate(self.task.iterate_result_candidates(self.num_samples, repeats=repeats)): subject = f" - Subject: {sample.subject}" sample_index = i + 1 if sample.result_id in subject_response_id_mapping.get(sample.subject, []): logger.info( f"Task: {self.response_type.value}{subject} - Sample: {sample_index} - skipping, already done." ) pbar.update(1) continue logger.info(f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}") pbar.set_postfix_str(f"Sample {sample_index}/{total_num_samples}") pbar.update(1) samples_batch.append(sample) if len(samples_batch) >= samples_batch_size: _process_batch(samples_batch) samples_batch = [] _process_batch(samples_batch) if not self.save_intermediate_results: self.result_processor.save_responses(responses) return responses def _get_metadata(self) -> dict[str, Any]: all_metrics = getattr(self.task, "METRICS", None) metadata = self.config.model_dump(mode="json") metadata["llm_name"] = self.llm.name metadata["task_name"] = self.task_name metadata["metrics"] = [m.NAME for m in all_metrics] if all_metrics is not None else [] metadata["primary_metrics"] = getattr(self.task, "PRIMARY_METRICS", None) metadata["eval_framework_version"] = eval_framework_version metadata["task_output_dir"] = str(self.result_processor.output_dir) if hasattr(self, "total_time"): metadata["start_time"] = str(datetime.fromtimestamp(self.start_time, UTC)) metadata["end_time"] = str(datetime.fromtimestamp(self.end_time, UTC)) metadata["total_time"] = self.total_time # add task specific metadata metadata["task_metadata"] = self.task.get_metadata() return metadata def _verify_loaded_metadata_compatibility(self) -> None: if not (loaded_metadata := self.result_processor.load_metadata()): return current_metadata = self._get_metadata() # check if crucial keys in metadata are the same as in the previous run keys = [ "task_name", "task_subjects", "num_fewshot", "num_samples", "llm_name", "llm_args", "repeats", ] for key in keys: if loaded_metadata.get(key) != current_metadata[key]: raise ValueError(f"Existing metadata does not match current metadata for {key}.") def __del__(self) -> None: self.llm.__del__()
[docs] def generate(self) -> list[Completion | Loglikelihood]: """Generates responses and saves them along with metadata.""" logger.info(f"{RED}[ Running responses generation ---------- ]{RESET}") logger.info(f"{RED}[ Will save into {self.result_processor.output_dir} ---------- ]{RESET}") responses = self._run_task_against_model() logger.info("Completions generated and saved.") return responses