Updated accuracy / precision tests (#3333)

Added metrics devices: cpu, cuda, mps if available

Author: vfdev-5

tests/ignite/__init__.py

@@ -1,11 +1,12 @@
 import torch
 
-
-def cpu_and_maybe_cuda():
-    return ("cpu",) + (("cuda",) if torch.cuda.is_available() else ())
+from ignite.distributed.comp_models.base import _torch_version_gt_112
 
 
 def is_mps_available_and_functional():
+    if not _torch_version_gt_112:
+        return False
+
     if not torch.backends.mps.is_available():
         return False
     try:

tests/ignite/conftest.py

@@ -13,6 +13,7 @@
 import torch.distributed as dist
 
 import ignite.distributed as idist
+from tests.ignite import is_mps_available_and_functional
 
 
 def pytest_addoption(parser):
@@ -72,6 +73,9 @@ def term_handler():
     params=[
         "cpu",
         pytest.param("cuda", marks=pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no CUDA support")),
+        pytest.param(
+            "mps", marks=pytest.mark.skipif(not is_mps_available_and_functional(), reason="Skip if no MPS support")
+        ),
     ]
 )
 def available_device(request):

tests/ignite/distributed/test_auto.py

@@ -12,7 +12,6 @@
 
 import ignite.distributed as idist
 from ignite.distributed.auto import auto_dataloader, auto_model, auto_optim, DistributedProxySampler
-from ignite.distributed.comp_models.base import _torch_version_gt_112
 from tests.ignite import is_mps_available_and_functional
 
 
@@ -181,10 +180,7 @@ def _test_auto_model_optimizer(ws, device):
         assert optimizer.backward_passes_per_step == backward_passes_per_step
 
 
-@pytest.mark.skipif(
-    (not _torch_version_gt_112) or (torch.backends.mps.is_available() and not is_mps_available_and_functional()),
-    reason="Skip if MPS not functional",
-)
+@pytest.mark.skipif(not is_mps_available_and_functional(), reason="Skip if MPS not functional")
 def test_auto_methods_no_dist():
     _test_auto_dataloader(1, 1, batch_size=1)
     _test_auto_dataloader(1, 1, batch_size=10, num_workers=2)

tests/ignite/distributed/test_launcher.py

@@ -8,7 +8,6 @@
 from packaging.version import Version
 
 import ignite.distributed as idist
-from ignite.distributed.comp_models.base import _torch_version_gt_112
 from ignite.distributed.utils import has_hvd_support, has_native_dist_support, has_xla_support
 from tests.ignite import is_mps_available_and_functional
 
@@ -56,10 +55,7 @@ def execute(cmd, env=None):
     return str(process.stdout.read()) + str(process.stderr.read())
 
 
-@pytest.mark.skipif(
-    (not _torch_version_gt_112) or (torch.backends.mps.is_available() and not is_mps_available_and_functional()),
-    reason="Skip if MPS not functional",
-)
+@pytest.mark.skipif(not is_mps_available_and_functional(), reason="Skip if MPS not functional")
 def test_check_idist_parallel_no_dist(exec_filepath):
     cmd = [sys.executable, "-u", exec_filepath]
     out = execute(cmd)

tests/ignite/engine/test_create_supervised.py

@@ -12,7 +12,6 @@
 from torch.optim import SGD
 
 import ignite.distributed as idist
-from ignite.distributed.comp_models.base import _torch_version_gt_112
 from ignite.engine import (
     _check_arg,
     create_supervised_evaluator,
@@ -488,7 +487,7 @@ def test_create_supervised_trainer_on_cuda():
     _test_create_mocked_supervised_trainer(model_device=model_device, trainer_device=trainer_device)
 
 
-@pytest.mark.skipif(not (_torch_version_gt_112 and is_mps_available_and_functional()), reason="Skip if no MPS")
+@pytest.mark.skipif(not is_mps_available_and_functional(), reason="Skip if no MPS")
 def test_create_supervised_trainer_on_mps():
     model_device = trainer_device = "mps"
     _test_create_supervised_trainer_wrong_accumulation(model_device=model_device, trainer_device=trainer_device)
@@ -669,14 +668,14 @@ def test_create_supervised_evaluator_on_cuda_with_model_on_cpu():
     _test_mocked_supervised_evaluator(evaluator_device="cuda")
 
 
-@pytest.mark.skipif(not (_torch_version_gt_112 and is_mps_available_and_functional()), reason="Skip if no MPS")
+@pytest.mark.skipif(not is_mps_available_and_functional(), reason="Skip if no MPS")
 def test_create_supervised_evaluator_on_mps():
     model_device = evaluator_device = "mps"
     _test_create_supervised_evaluator(model_device=model_device, evaluator_device=evaluator_device)
     _test_mocked_supervised_evaluator(model_device=model_device, evaluator_device=evaluator_device)
 
 
-@pytest.mark.skipif(not (_torch_version_gt_112 and is_mps_available_and_functional()), reason="Skip if no MPS")
+@pytest.mark.skipif(not is_mps_available_and_functional(), reason="Skip if no MPS")
 def test_create_supervised_evaluator_on_mps_with_model_on_cpu():
     _test_create_supervised_evaluator(evaluator_device="mps")
     _test_mocked_supervised_evaluator(evaluator_device="mps")

tests/ignite/metrics/conftest.py

@@ -0,0 +1,83 @@
+import pytest
+import torch
+
+
+@pytest.fixture(params=range(14))
+def test_data_binary(request):
+    return [
+        # Binary accuracy on input of shape (N, 1) or (N, )
+        (torch.randint(0, 2, size=(10,)), torch.randint(0, 2, size=(10,)), 1),
+        (torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)), 1),
+        # updated batches
+        (torch.randint(0, 2, size=(50,)), torch.randint(0, 2, size=(50,)), 16),
+        (torch.randint(0, 2, size=(50, 1)), torch.randint(0, 2, size=(50, 1)), 16),
+        # Binary accuracy on input of shape (N, L)
+        (torch.randint(0, 2, size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1),
+        (torch.randint(0, 2, size=(10, 1, 5)), torch.randint(0, 2, size=(10, 1, 5)), 1),
+        # updated batches
+        (torch.randint(0, 2, size=(50, 5)), torch.randint(0, 2, size=(50, 5)), 16),
+        (torch.randint(0, 2, size=(50, 1, 5)), torch.randint(0, 2, size=(50, 1, 5)), 16),
+        # Binary accuracy on input of shape (N, H, W)
+        (torch.randint(0, 2, size=(10, 12, 10)), torch.randint(0, 2, size=(10, 12, 10)), 1),
+        (torch.randint(0, 2, size=(10, 1, 12, 10)), torch.randint(0, 2, size=(10, 1, 12, 10)), 1),
+        # updated batches
+        (torch.randint(0, 2, size=(50, 12, 10)), torch.randint(0, 2, size=(50, 12, 10)), 16),
+        (torch.randint(0, 2, size=(50, 1, 12, 10)), torch.randint(0, 2, size=(50, 1, 12, 10)), 16),
+        # Corner case with all zeros predictions
+        (torch.zeros(size=(10,), dtype=torch.long), torch.randint(0, 2, size=(10,)), 1),
+        (torch.zeros(size=(10, 1), dtype=torch.long), torch.randint(0, 2, size=(10, 1)), 1),
+    ][request.param]
+
+
+@pytest.fixture(params=range(14))
+def test_data_multiclass(request):
+    return [
+        # Multiclass input data of shape (N, ) and (N, C)
+        (torch.rand(10, 6), torch.randint(0, 6, size=(10,)), 1),
+        (torch.rand(10, 4), torch.randint(0, 4, size=(10,)), 1),
+        # updated batches
+        (torch.rand(50, 6), torch.randint(0, 6, size=(50,)), 16),
+        (torch.rand(50, 4), torch.randint(0, 4, size=(50,)), 16),
+        # Multiclass input data of shape (N, L) and (N, C, L)
+        (torch.rand(10, 5, 8), torch.randint(0, 5, size=(10, 8)), 1),
+        (torch.rand(10, 8, 12), torch.randint(0, 8, size=(10, 12)), 1),
+        # updated batches
+        (torch.rand(50, 5, 8), torch.randint(0, 5, size=(50, 8)), 16),
+        (torch.rand(50, 8, 12), torch.randint(0, 8, size=(50, 12)), 16),
+        # Multiclass input data of shape (N, H, W, ...) and (N, C, H, W, ...)
+        (torch.rand(10, 5, 18, 16), torch.randint(0, 5, size=(10, 18, 16)), 1),
+        (torch.rand(10, 7, 20, 12), torch.randint(0, 7, size=(10, 20, 12)), 1),
+        # updated batches
+        (torch.rand(50, 5, 18, 16), torch.randint(0, 5, size=(50, 18, 16)), 16),
+        (torch.rand(50, 7, 20, 12), torch.randint(0, 7, size=(50, 20, 12)), 16),
+        # Corner case with all zeros predictions
+        (torch.zeros(size=(10, 6)), torch.randint(0, 6, size=(10,)), 1),
+        (torch.zeros(size=(10, 4)), torch.randint(0, 4, size=(10,)), 1),
+    ][request.param]
+
+
+@pytest.fixture(params=range(14))
+def test_data_multilabel(request):
+    return [
+        # Multilabel input data of shape (N, C)
+        (torch.randint(0, 2, size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1),
+        (torch.randint(0, 2, size=(10, 4)), torch.randint(0, 2, size=(10, 4)), 1),
+        # updated batches
+        (torch.randint(0, 2, size=(50, 5)), torch.randint(0, 2, size=(50, 5)), 16),
+        (torch.randint(0, 2, size=(50, 4)), torch.randint(0, 2, size=(50, 4)), 16),
+        # Multilabel input data of shape (N, C, L)
+        (torch.randint(0, 2, size=(10, 5, 10)), torch.randint(0, 2, size=(10, 5, 10)), 1),
+        (torch.randint(0, 2, size=(10, 4, 10)), torch.randint(0, 2, size=(10, 4, 10)), 1),
+        # updated batches
+        (torch.randint(0, 2, size=(50, 5, 10)), torch.randint(0, 2, size=(50, 5, 10)), 16),
+        (torch.randint(0, 2, size=(50, 4, 10)), torch.randint(0, 2, size=(50, 4, 10)), 16),
+        # Multilabel input data of shape (N, C, H, W)
+        (torch.randint(0, 2, size=(10, 5, 18, 16)), torch.randint(0, 2, size=(10, 5, 18, 16)), 1),
+        (torch.randint(0, 2, size=(10, 4, 20, 23)), torch.randint(0, 2, size=(10, 4, 20, 23)), 1),
+        # updated batches
+        (torch.randint(0, 2, size=(50, 5, 18, 16)), torch.randint(0, 2, size=(50, 5, 18, 16)), 16),
+        (torch.randint(0, 2, size=(50, 4, 20, 23)), torch.randint(0, 2, size=(50, 4, 20, 23)), 16),
+        # Corner case with all zeros predictions
+        (torch.zeros(size=(10, 5)), torch.randint(0, 2, size=(10, 5)), 1),
+        (torch.zeros(size=(10, 4)), torch.randint(0, 2, size=(10, 4)), 1),
+    ][request.param]