Source code for eval_framework.tasks.registry

import contextlib
import re
from collections.abc import Generator, Iterator
from typing import Any

from eval_framework.contract import Benchmark
from eval_framework.tasks.lazy import Lazy

__all__ = [
    "Benchmark",
    "Lazy",
    "Registry",
    "with_registry",
    "is_registered",
    "validate_task_name",
    "registered_task_names",
]


[docs] class Registry: """A registry for Benchmarks""" def __init__(self) -> None: self._registry: dict[str, Benchmark] = dict() def __iter__(self) -> Iterator[str]: """Iterate over all task names in the registry.""" for factory in self._registry.values(): yield factory.id()
[docs] def task_names(self) -> list[str]: """The names of all registered tasks.""" return list(self)
[docs] def items(self) -> Iterator[tuple[str, Benchmark]]: """Iterate over `(task name, Benchmark)` pairs in the registry.""" for factory in self._registry.values(): yield factory.id(), factory
@staticmethod def _task_key(name: str, /) -> str: name = re.sub(r"[\s\-_]+", "", name).upper() if not name.isalnum(): raise ValueError( f"Task name '{name}' contains invalid characters. Only alphanumeric characters are allowed." ) return name def __contains__(self, name: str) -> bool: task_key = self._task_key(name) return task_key in self._registry def __getitem__(self, name: str, /) -> Benchmark: task_key = self._task_key(name) try: return self._registry[task_key] except KeyError: raise KeyError(f"Task not found: {name=} with task_key {task_key=}")
[docs] def add(self, factory: Benchmark) -> None: """Register a factory under the key derived from its ``id()``.""" task_key = self._task_key(factory.id()) if task_key in self._registry: raise ValueError(f"Cannot register duplicate task with key: {task_key}") self._registry[task_key] = factory
_REGISTRY = Registry() def registry() -> Registry: return _REGISTRY
[docs] @contextlib.contextmanager def with_registry(registry: Registry) -> Generator[None, Any, None]: """Contextmanager to change the current registry.""" global _REGISTRY old_registry = _REGISTRY try: _REGISTRY = registry yield finally: _REGISTRY = old_registry
[docs] def registered_task_names() -> list[str]: """Return the names of all registered tasks.""" return list(_REGISTRY)
[docs] def is_registered(name: str, /) -> bool: """Return True if a task is registered.""" return name in _REGISTRY
[docs] def validate_task_name(name: str) -> str: """Pydantic-style validator for task names.""" if name not in registry(): raise ValueError(f"Task not registered: {name}") return name