Source code for eval_framework.tasks.base

# The companion still imports ``Language`` from here; the alias form keeps it a re-export
# under ``no_implicit_reexport``.
from eval_framework.tasks.task_style import Language as Language

RANDOM_SEED = 42


def _subject_parts(subject: object) -> tuple[str, ...]:
    """A subject as its stringified parts: tuple subjects keep their fields, scalars are a single part.

    Comparing string forms means tuple fields of any type work (e.g. tuple[str, int, str]) without
    having to parse the CLI token into each field's native type.
    """
    return tuple(str(field) for field in subject) if isinstance(subject, tuple) else (str(subject),)


[docs] def resolve_overwrite_subjects[SubjectType]( custom_subjects: list[str], accepted_subjects: list[SubjectType], task_name: str ) -> list[SubjectType]: """Restrict `accepted_subjects` to the ones requested via --task-subjects. A subject matches a token when their parts agree position by position, where "*" matches any one part -- so "DE_DE,*" selects every German subject and "*" selects every scalar subject. """ if not accepted_subjects: raise ValueError(f"Task {task_name} has no SUBJECTS defined") filters = {token: tuple(part.strip() for part in token.split(",")) for token in custom_subjects} # Select and validate in one pass. Every filter is tried against every subject (no early exit), so # a filter still counts as used when its subjects were already selected by an earlier filter -- # e.g. "a,*" after "a,1" is valid, not unused. chosen_subjects: list[SubjectType] = [] used_filters: set[str] = set() for subject in accepted_subjects: fields = _subject_parts(subject) matching = [ token for token, parts in filters.items() if len(parts) == len(fields) and all(p in ("*", f) for p, f in zip(parts, fields)) ] if matching: chosen_subjects.append(subject) used_filters.update(matching) for token in custom_subjects: if token not in used_filters: raise ValueError( f"Subject '{token}' not found in task {task_name}. Subjects are matched by their " f"string form, so check number and enum formatting." ) return chosen_subjects