eval_framework.metrics.efficiency package

Submodules

eval_framework.metrics.efficiency.bytes_per_sequence_position module

class eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesCompletion

Bases: BaseMetric[Completion]

NAME: str = 'Bytes'

calculate(response)

Return type:

list[MetricResult]

Parameters:

response (Completion)

class eval_framework.metrics.efficiency.bytes_per_sequence_position.BytesLoglikelihood

Bases: BaseMetric[Loglikelihood]

NAME: str = 'Bytes'

calculate(response)

Return type:

list[MetricResult]

Parameters:

response (Loglikelihood)

class eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsCompletion

Bases: BaseMetric[Completion]

NAME: str = 'SequencePositions'

calculate(response)

Return type:

list[MetricResult]

Parameters:

response (Completion)

class eval_framework.metrics.efficiency.bytes_per_sequence_position.SequencePositionsLoglikelihood

Bases: BaseMetric[Loglikelihood]

NAME: str = 'SequencePositions'

calculate(response)

Return type:

list[MetricResult]

Parameters:

response (Loglikelihood)

eval_framework.metrics.efficiency.token_counters module

class eval_framework.metrics.efficiency.token_counters.TokenCounts

Bases: BaseMetric[Completion]

Number of tokens the model generated for the completion, and how many of those were spent on reasoning (thinking).

Reads the token counts backends already attach to the response, so no extra tokenisation is performed. Each value independently falls back to None when the backend did not report that particular count or when the sample errored. Reasoning counts are only exposed by some backends (e.g. OpenAI reasoning models via usage.completion_tokens_details.reasoning_tokens, or vLLM started with –reasoning-parser); non-reasoning models and backends that do not surface a per-response count return None for that key.

KEYS: list[str] | None = ['Completion', 'Reasoning']

NAME: str = 'TokenCounts'

calculate(response)

Return type:

list[MetricResult]

Parameters:

response (Completion)

Module contents