import importlib.resources
from collections.abc import Callable
from typing import Self
from llm_sandbox.exceptions import SandboxTimeoutError
from pydantic import ConfigDict
from eval_framework.metrics.base import BaseMetric, MetricResult
from eval_framework.shared.types import (
BaseMetricContext,
Completion,
extract_context_metric,
)
from eval_framework.tasks.utils import (
CallableSerializer,
ExecutionResult,
execute_python_code_with_tests,
)
[docs]
class CodeExecutionBaseContext(BaseMetricContext):
model_config = ConfigDict(use_attribute_docstrings=True)
run_env: str
"""Name of docker image to run unit-tests inside."""
code_prompt: str
"""Prompt to LLM for code generation."""
test_code: str
"""Python code that contains logic for unit test execution."""
benchmark_timeout: int = 60
"""Time in seconds for the full test execution run."""
package_downloads: dict[str, str | None]
"""A dictionary listing the packages and their respective names in PyPi into the LLM sandbox."""
[docs]
class CodeExecutionPassAtOneContext(CodeExecutionBaseContext):
snippet_merge_fn: str
"""Logic for merging LLM generated code with test execution code; this code will be passed into the sandbox
to run the testing process. This code is serialized."""
output_parse_fn: str
"""Logic for parsing the output of test code execution run within the LLM sandbox. This code is serialized."""
[docs]
class RealtimeCodeExectionContext(CodeExecutionBaseContext):
snippet_merge_fn: Callable[[str, str], str]
"""Logic for merging LLM generated code with test execution code; this code will be passed into the sandbox
to run the testing process. This code is deserialized."""
output_parse_fn: Callable[[str], ExecutionResult]
"""Logic for parsing the output of test code execution run within the LLM sandbox. This code is deserialized."""
[docs]
@classmethod
def from_context(cls, context: CodeExecutionPassAtOneContext) -> Self:
return cls(
run_env=context.run_env,
code_prompt=context.code_prompt,
test_code=context.test_code,
benchmark_timeout=context.benchmark_timeout,
snippet_merge_fn=CallableSerializer.decode(context.snippet_merge_fn),
output_parse_fn=CallableSerializer.decode(context.output_parse_fn),
package_downloads=context.package_downloads,
)
[docs]
class CodeExecutionPassAtOne(BaseMetric[Completion]):
NAME = "code-execution-pass@1"
def __init__(self) -> None:
self.k = 1
# NOTE : this serializer should be the same class as initialized in the benchmark
self.serializer = CallableSerializer()
[docs]
def calculate(self, response: Completion) -> list[MetricResult]:
try:
context = extract_context_metric(response, CodeExecutionPassAtOneContext)
parsed_context = RealtimeCodeExectionContext.from_context(context)
except Exception as e:
raise Exception(f"Failed to rebuild parsing functions => {e}")
n = 1 # we only support N=1 at the moment
# A code timeout is scored as a failure inside _count_correct_samples; an infra failure
# (e.g. an image pull rate limit) propagates and aborts the run.
c, output = self._count_correct_samples(response.completion, parsed_context)
pass_at_k_value = estimate_pass_at_k(n, c, self.k)
return [
MetricResult(
metric_name=self.NAME,
value=pass_at_k_value,
higher_is_better=True,
code_execution_trace=output,
)
]
def _count_correct_samples(self, completion: str, context: RealtimeCodeExectionContext) -> tuple[int, str]:
try:
result = execute_python_code_with_tests(
code=completion,
test_code=context.test_code,
package_mapping=context.package_downloads,
merge_code_fn=context.snippet_merge_fn,
image=context.run_env,
timeout=context.benchmark_timeout,
parse_output_fn=context.output_parse_fn,
dockerfile=None,
)
except SandboxTimeoutError as e:
# The submitted code timed out (e.g. an infinite loop) -- a failing sample.
return (0, str(e))
return (1 if result.success else 0), result.output
[docs]
class CodeExecutionPassAtOneWithCodebench(CodeExecutionPassAtOne):
NAME = "code-execution-pass@1-codebench"
def __init__(self) -> None:
super().__init__()
self.dockerfile = str(importlib.resources.files("eval_framework.tasks") / "Dockerfile_codebench")
def _count_correct_samples(self, completion: str, context: RealtimeCodeExectionContext) -> tuple[int, str]:
try:
result = execute_python_code_with_tests(
code=completion,
test_code=context.test_code,
package_mapping={}, # the docker contains everything
merge_code_fn=context.snippet_merge_fn,
image=None, # dockerfile provided
timeout=context.benchmark_timeout,
parse_output_fn=context.output_parse_fn,
dockerfile=self.dockerfile,
)
except SandboxTimeoutError as e:
# The submitted code timed out (e.g. an infinite loop) -- a failing sample.
return (0, str(e))
return (1 if result.success else 0), result.output
[docs]
def estimate_pass_at_k(n: int, c: int, k: int) -> float:
"""
Estimates pass@k for a single problem.
Parameters:
n (int): Total number of generated samples.
c (int): Number of correct samples.
k (int): Number of attempts or samples considered.
Returns:
float: The pass@k value.
"""
if n - c < k:
return 1.0
# Calculate the probability that at least one of the k samples is correct
probability = 1.0
for i in range(k):
probability *= (n - c - i) / (n - i)
return 1.0 - probability