import logging
import time
from collections.abc import Callable
from datetime import UTC, datetime
from functools import partial
from typing import Any
from tqdm import tqdm
from eval_framework import __version__ as eval_framework_version
from eval_framework.contract import Benchmark, ResponseType, ResultCandidate
from eval_framework.llm.base import BaseLLM
from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
from eval_framework.result_processors.result_processor import ResultsFileProcessor
from eval_framework.shared.types import (
Completion,
Loglikelihood,
RawLoglikelihood,
)
from eval_framework.tasks.base import RANDOM_SEED
from eval_framework.tasks.eval_config import EvalConfig
from eval_framework.tasks.registry import Registry, registry
from eval_framework.utils.constants import RED, RESET
from eval_framework.utils.tqdm_handler import get_disable_bar_flag
logger = logging.getLogger(__name__)
def _require_judge_when_task_uses_one(benchmark: Benchmark, config: EvalConfig) -> None:
"""A task scored by an LLM-judge metric needs a judge model; fail early if the config lacks one."""
if config.llm_judge_class is not None:
return
if any(issubclass(metric, BaseLLMJudgeMetric) for metric in benchmark.metrics()):
raise ValueError(
f"Task '{config.task_name}' is scored with an LLM-judge metric, so the eval config must set "
"llm_judge_class."
)
[docs]
class ResponseGenerator:
def __init__(
self,
llm: BaseLLM,
config: EvalConfig,
result_processor: ResultsFileProcessor,
*,
benchmark_registry: Registry | None = None,
) -> None:
self.task_name = config.task_name
self.llm = llm
self.config = config
self.result_processor = result_processor
self.num_samples = config.num_samples
self.save_intermediate_results = config.save_intermediate_results
benchmark = (benchmark_registry if benchmark_registry is not None else registry())[config.task_name]
_require_judge_when_task_uses_one(benchmark, config)
self.task = benchmark.create(
config.num_fewshot,
config.task_subjects,
config.hf_revision,
seed=RANDOM_SEED,
)
self.response_type = self.task.get_response_type()
def _llm_task_param_precedence(self) -> tuple[list[str] | None, int | None]:
llm_stop_sequences = getattr(self.llm, "stop_sequences", None)
llm_max_tokens = getattr(self.llm, "max_tokens", None)
task_stop_sequences = self.task.get_stop_sequences()
task_max_tokens = self.config.max_tokens or self.task.get_max_tokens()
# if both task and model define a max_token, the smaller value is used
max_tokens = min(
[x for x in [llm_max_tokens, task_max_tokens] if x is not None],
default=None,
)
logger.info(f"Set max_tokens to {max_tokens}")
# if both task and model define stop sequences, those are merged into one list
stop_sequences_merged = (llm_stop_sequences or []) + task_stop_sequences
stop_sequences = sorted(list(set(stop_sequences_merged))) if stop_sequences_merged else None
logger.info(f"Set stop_sequences to {stop_sequences}")
return stop_sequences, max_tokens
def _generate_loglikelihoods(self, samples: list[ResultCandidate]) -> list[Loglikelihood]:
raw_loglikelihoods: list[RawLoglikelihood]
raw_loglikelihoods = self.llm.logprobs(samples)
loglikelihood_list = []
for idx, sample in enumerate(samples):
raw_loglikelihood = raw_loglikelihoods[idx]
assert sample.ground_truth is not None
loglikelihood_list.append(
Loglikelihood(
sample_id=sample.sample_id,
result_id=sample.result_id,
subject=sample.subject,
ground_truth=sample.ground_truth,
prompt=raw_loglikelihood.prompt,
prompt_num_tokens=raw_loglikelihood.prompt_num_tokens,
concat_compression=raw_loglikelihood.concat_compression,
loglikelihoods=raw_loglikelihood.loglikelihoods,
loglikelihoods_num_tokens=raw_loglikelihood.loglikelihoods_num_tokens,
loglikelihoods_per_token=raw_loglikelihood.loglikelihoods_per_token,
context=sample.context,
)
)
return loglikelihood_list
def _generative_output_type_selector(
self,
) -> Callable[[list[ResultCandidate]], list[Completion] | list[Loglikelihood]]:
match self.response_type:
case ResponseType.COMPLETION:
stop_sequences, max_tokens = self._llm_task_param_precedence()
return partial(
self.task.generate_completions,
self.llm,
stop_sequences=stop_sequences,
max_tokens=max_tokens,
)
case ResponseType.LOGLIKELIHOODS:
return self._generate_loglikelihoods
case _:
raise KeyError(f"Task type {self.task} not supported")
def _run_task_against_model(self) -> list[Completion | Loglikelihood]:
logger.info(f"{RED}[ Running task {self.task.display_name()} against model ------------ ]{RESET}")
self.start_time, monotonic_start = time.time(), time.monotonic()
run_fn = self._generative_output_type_selector()
self._verify_loaded_metadata_compatibility()
responses = self.result_processor.load_responses() # load responses if present
subject_response_id_mapping = self._map_subject_response_ids(responses)
self.result_processor.save_metadata(self._get_metadata())
responses = self._curate_responses(responses, subject_response_id_mapping, run_fn)
self.end_time, monotonic_end = time.time(), time.monotonic()
self.total_time = monotonic_end - monotonic_start
self.result_processor.save_metadata(self._get_metadata()) # overwrite with updated timing
return responses
def _map_subject_response_ids(self, responses: list[Completion | Loglikelihood]) -> dict[str, set[int]]:
subject_response_id_mapping = {}
if responses:
response_subjects = {resp.subject for resp in responses}
subject_response_id_mapping = {
response_subject: set([resp.result_id for resp in responses if resp.subject == response_subject])
for response_subject in response_subjects
}
return subject_response_id_mapping
def _curate_responses(
self,
responses: list[Completion | Loglikelihood],
subject_response_id_mapping: dict[str, set[int]],
generative_output_function: Callable[[list[ResultCandidate]], list[Completion] | list[Loglikelihood]],
) -> list[Completion | Loglikelihood]:
def _process_batch(samples_batch: list[ResultCandidate]) -> None:
if not samples_batch:
return
if len(samples_batch) > 1:
logger.info("Processing batch...")
responses_batch = generative_output_function(samples_batch)
responses.extend(responses_batch)
if self.save_intermediate_results:
for response in responses_batch:
self.result_processor.save_response(response)
# In order to enable parallelism we group samples in batches and send them in parallel to the `run_fn`.
# The BaseLLM class is then in charge of managing the parallelism (eg, using AsyncClient in API models).
# If samples_batch_size = 1, samples are run sequentially; in any case, we return here after finishing each
# individual batch to save cached results.
samples_batch_size = self.config.batch_size
repeats = self.config.repeats
# Calculate total samples for progress bar - use num_samples or iterate to count
if self.num_samples is None:
# Count samples by iterating (this might be expensive for large datasets)
total_num_samples = sum(1 for _ in self.task.iterate_result_candidates(None, repeats=repeats))
else:
total_num_samples = self.num_samples * repeats
samples_batch: list[ResultCandidate] = []
with tqdm(
total=total_num_samples,
desc=f"Processing {self.response_type.value}",
disable=get_disable_bar_flag(),
) as pbar:
for i, sample in enumerate(self.task.iterate_result_candidates(self.num_samples, repeats=repeats)):
subject = f" - Subject: {sample.subject}"
sample_index = i + 1
if sample.result_id in subject_response_id_mapping.get(sample.subject, []):
logger.info(
f"Task: {self.response_type.value}{subject} - Sample: {sample_index} - skipping, already done."
)
pbar.update(1)
continue
logger.info(f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}")
pbar.set_postfix_str(f"Sample {sample_index}/{total_num_samples}")
pbar.update(1)
samples_batch.append(sample)
if len(samples_batch) >= samples_batch_size:
_process_batch(samples_batch)
samples_batch = []
_process_batch(samples_batch)
if not self.save_intermediate_results:
self.result_processor.save_responses(responses)
return responses
def _get_metadata(self) -> dict[str, Any]:
all_metrics = getattr(self.task, "METRICS", None)
metadata = self.config.model_dump(mode="json")
metadata["llm_name"] = self.llm.name
metadata["task_name"] = self.task_name
metadata["metrics"] = [m.NAME for m in all_metrics] if all_metrics is not None else []
metadata["primary_metrics"] = getattr(self.task, "PRIMARY_METRICS", None)
metadata["eval_framework_version"] = eval_framework_version
metadata["task_output_dir"] = str(self.result_processor.output_dir)
if hasattr(self, "total_time"):
metadata["start_time"] = str(datetime.fromtimestamp(self.start_time, UTC))
metadata["end_time"] = str(datetime.fromtimestamp(self.end_time, UTC))
metadata["total_time"] = self.total_time
# add task specific metadata
metadata["task_metadata"] = self.task.get_metadata()
return metadata
def _verify_loaded_metadata_compatibility(self) -> None:
if not (loaded_metadata := self.result_processor.load_metadata()):
return
current_metadata = self._get_metadata()
# check if crucial keys in metadata are the same as in the previous run
keys = [
"task_name",
"task_subjects",
"num_fewshot",
"num_samples",
"llm_name",
"llm_args",
"repeats",
]
for key in keys:
if loaded_metadata.get(key) != current_metadata[key]:
raise ValueError(f"Existing metadata does not match current metadata for {key}.")
def __del__(self) -> None:
self.llm.__del__()
[docs]
def generate(self) -> list[Completion | Loglikelihood]:
"""Generates responses and saves them along with metadata."""
logger.info(f"{RED}[ Running responses generation ---------- ]{RESET}")
logger.info(f"{RED}[ Will save into {self.result_processor.output_dir} ---------- ]{RESET}")
responses = self._run_task_against_model()
logger.info("Completions generated and saved.")
return responses