Source code for eval_framework.metrics.efficiency.bytes_per_sequence_position

from eval_framework.metrics.base import BaseMetric, MetricResult
from eval_framework.shared.types import Completion, Loglikelihood



[docs]
class BytesLoglikelihood(BaseMetric[Loglikelihood]):
    NAME = "Bytes"


[docs]
    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
        if response.error or response.concat_compression is None:
            value = None
        else:
            value = response.concat_compression.num_bytes

        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]





[docs]
class SequencePositionsLoglikelihood(BaseMetric[Loglikelihood]):
    NAME = "SequencePositions"


[docs]
    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
        if response.error or response.concat_compression is None:
            value = None
        else:
            value = response.concat_compression.num_tokens
        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]





[docs]
class BytesCompletion(BaseMetric[Completion]):
    NAME = "Bytes"


[docs]
    def calculate(self, response: Completion) -> list[MetricResult]:
        if response.error or response.concat_compression is None:
            value = None
        else:
            value = response.concat_compression.num_bytes

        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]





[docs]
class SequencePositionsCompletion(BaseMetric[Completion]):
    NAME = "SequencePositions"


[docs]
    def calculate(self, response: Completion) -> list[MetricResult]:
        if response.error or response.concat_compression is None:
            value = None
        else:
            value = response.concat_compression.num_tokens
        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]