From 1fcd8d836ce026d437a188e0b023f36b778c48f1 Mon Sep 17 00:00:00 2001 From: Alex-Andrei Cuvuliuc Date: Mon, 13 Jul 2026 12:11:07 +0200 Subject: [PATCH] correctly evaluate the score_direction argument --- metaquantus/helpers/configs_utils.py | 2 +- metaquantus/meta_evaluation.py | 7 ++++++- tests/test_meta_evaluation.py | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/metaquantus/helpers/configs_utils.py b/metaquantus/helpers/configs_utils.py index 49d9c4b..3412e57 100644 --- a/metaquantus/helpers/configs_utils.py +++ b/metaquantus/helpers/configs_utils.py @@ -8,7 +8,7 @@ class Estimator: name: str category: str - score_direction: bool + score_direction: str init: quantus.Metric diff --git a/metaquantus/meta_evaluation.py b/metaquantus/meta_evaluation.py index f0a9fba..5601ae1 100644 --- a/metaquantus/meta_evaluation.py +++ b/metaquantus/meta_evaluation.py @@ -780,11 +780,16 @@ def compute_iec_adversary( ------- float """ + if score_direction not in ("lower", "higher"): + raise ValueError( + f"score_direction must be 'lower' or 'higher', got {score_direction!r}." + ) + U = [] for row_star, row_hat in zip(Q_star, Q_hat): for q_star, q_hat in zip(row_star[indices], row_hat[indices]): - if score_direction: + if score_direction == "lower": if q_star < q_hat: U.append(1) else: diff --git a/tests/test_meta_evaluation.py b/tests/test_meta_evaluation.py index f249851..aeee8fb 100644 --- a/tests/test_meta_evaluation.py +++ b/tests/test_meta_evaluation.py @@ -79,7 +79,7 @@ def test_meta_evaluation_mnist( iters = 3 K = 5 metric = estimators[estimator_category][estimator_name]["init"] - score_direction = estimators[estimator_category][estimator_name]["init"] + score_direction = estimators[estimator_category][estimator_name]["score_direction"] # Define the meta-evaluation exercise. meta_evaluator = MetaEvaluation( @@ -178,7 +178,7 @@ def test_meta_evaluation_fmnist( iters = 3 K = 5 metric = estimators[estimator_category][estimator_name]["init"] - score_direction = estimators[estimator_category][estimator_name]["init"] + score_direction = estimators[estimator_category][estimator_name]["score_direction"] # Define the meta-evaluation exercise. meta_evaluator = MetaEvaluation( @@ -279,7 +279,7 @@ def test_meta_evaluation_cmnist( iters = 5 K = 10 metric = estimators[estimator_category][estimator_name]["init"] - score_direction = estimators[estimator_category][estimator_name]["init"] + score_direction = estimators[estimator_category][estimator_name]["score_direction"] # Define the meta-evaluation exercise. meta_evaluator = MetaEvaluation( @@ -311,4 +311,4 @@ def test_meta_evaluation_cmnist( ((scores_1 >= expected["min"]) & (scores_1 <= expected["max"]) & (scores_2 >= expected["min"]) & (scores_2 <= expected["max"])) ), "Test failed." -''' \ No newline at end of file +'''