From b34a98ac072d512b8c4999a605a6830b3cc3987e Mon Sep 17 00:00:00 2001 From: 72004 Date: Sun, 18 Oct 2026 18:23:59 +0500 Subject: [PATCH] AC: respect calculate_average in multi_accuracy and f1-score With calculate_average: False, f1-score still dropped the last element of the precision/recall results as if it were the average, so the last class was missing from the output. multi_accuracy ignored the option and always appended the average, which did not match its meta names. --- .../metrics/multilabel_recognition.py | 8 ++- .../tests/test_multilabel_metrics.py | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tools/accuracy_checker/tests/test_multilabel_metrics.py diff --git a/tools/accuracy_checker/accuracy_checker/metrics/multilabel_recognition.py b/tools/accuracy_checker/accuracy_checker/metrics/multilabel_recognition.py index be9b73c0716..d4e7fc1b07c 100644 --- a/tools/accuracy_checker/accuracy_checker/metrics/multilabel_recognition.py +++ b/tools/accuracy_checker/accuracy_checker/metrics/multilabel_recognition.py @@ -134,6 +134,8 @@ def update(self, annotation, prediction): def evaluate(self, annotations, predictions): tp_tn = np.add(self.tp, self.tn, dtype=float) per_class = np.divide(tp_tn, self.counter, out=np.zeros_like(tp_tn, dtype=float), where=self.counter != 0) + if not self.calculate_average: + return per_class average = np.sum(tp_tn) / np.sum(self.counter) return [*per_class, average] @@ -230,9 +232,11 @@ def update(self, annotation, prediction): def evaluate(self, annotations, predictions): precisions = self.precision.evaluate(annotations, predictions) recalls = self.recall.evaluate(annotations, predictions) + # precision and recall share this config, so they only end with an average if calculate_average is set + num_classes = len(self.labels) - precision_add = np.add(precisions[:-1], recalls[:-1], dtype=float) - precision_multiply = np.multiply(precisions[:-1], recalls[:-1], dtype=float) + precision_add = np.add(precisions[:num_classes], recalls[:num_classes], dtype=float) + precision_multiply = np.multiply(precisions[:num_classes], recalls[:num_classes], dtype=float) per_class = 2 * np.divide( precision_multiply, precision_add, out=np.zeros_like(precision_multiply, dtype=float), diff --git a/tools/accuracy_checker/tests/test_multilabel_metrics.py b/tools/accuracy_checker/tests/test_multilabel_metrics.py new file mode 100644 index 00000000000..e04ca5da095 --- /dev/null +++ b/tools/accuracy_checker/tests/test_multilabel_metrics.py @@ -0,0 +1,61 @@ +""" +Copyright (c) 2026 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +import pytest +from accuracy_checker.metrics.multilabel_recognition import F1Score, MultiLabelAccuracy +from accuracy_checker.representation import MultiLabelRecognitionAnnotation, MultiLabelRecognitionPrediction +from .common import DummyDataset + + +LABEL_MAP = {0: 'a', 1: 'b', 2: 'c'} + + +def evaluate(metric_cls, config): + metric = metric_cls(config, DummyDataset(label_map=LABEL_MAP)) + annotations = [ + MultiLabelRecognitionAnnotation('img1', [1, 0, 1]), + MultiLabelRecognitionAnnotation('img2', [0, 1, 1]), + ] + predictions = [ + MultiLabelRecognitionPrediction('img1', [1, 1, 1]), + MultiLabelRecognitionPrediction('img2', [0, 1, 0]), + ] + for annotation, prediction in zip(annotations, predictions): + metric.update(annotation, prediction) + return list(metric.evaluate(annotations, predictions)), metric.meta['names'] + + +class TestMultiLabelMetrics: + def test_f1_score_with_average(self): + result, names = evaluate(F1Score, {'type': 'f1-score'}) + assert names == ['a', 'b', 'c', 'average'] + assert result == pytest.approx([1.0, 2 / 3, 2 / 3, 0.75]) + + def test_f1_score_without_average_keeps_all_classes(self): + result, names = evaluate(F1Score, {'type': 'f1-score', 'calculate_average': False}) + assert names == ['a', 'b', 'c'] + assert result == pytest.approx([1.0, 2 / 3, 2 / 3]) + + def test_multi_accuracy_with_average(self): + result, names = evaluate(MultiLabelAccuracy, {'type': 'multi_accuracy'}) + assert names == ['a', 'b', 'c', 'average'] + assert result == pytest.approx([1.0, 0.5, 0.5, 2 / 3]) + + def test_multi_accuracy_without_average(self): + result, names = evaluate(MultiLabelAccuracy, {'type': 'multi_accuracy', 'calculate_average': False}) + assert names == ['a', 'b', 'c'] + assert result == pytest.approx([1.0, 0.5, 0.5])