Updated metrics and tests to run on distributed configuration

- Test on 2 GPUS single node - Added cmd in .travis.yml to indicate how to test locally - Updated travis to run tests in 4 processes
pytorch · vfdev-5 · Oct 24, 2019 · Aug 1, 2019 · Aug 1, 2019 · Aug 1, 2019
commit e7d12d038359ef9fd26a6533486e920e1e651835
diff --git a/.travis.yml b/.travis.yml
@@ -43,7 +43,7 @@ install:
   - pip install gym==0.10.11
 
 script:
-  - py.test --cov ignite --cov-report term-missing
+  - CUDA_VISIBLE_DEVICES="" py.test --tx 4*popen//python=python$TRAVIS_PYTHON_VERSION --cov ignite --cov-report term-missing -vvv tests/
 
   # Smoke tests for the examples
   # Mnist
@@ -73,7 +73,6 @@ script:
 
   # tests for distributed ops
   # As no GPUs on travis -> all tests will be skipped
-  - pip install pytest-xdist
   # 2 is the number of processes <-> number of available GPUs
   - export WORLD_SIZE=2
   - pytest --dist=each --tx $WORLD_SIZE*popen//python=python$TRAVIS_PYTHON_VERSION tests -m distributed -vvv

diff --git a/ignite/metrics/accumulation.py b/ignite/metrics/accumulation.py
@@ -6,6 +6,7 @@
 
 import torch
 
+
 class VariableAccumulation(Metric):
     """Single variable accumulator helper to compute (arithmetic, geometric, harmonic) average of a single variable.
 

diff --git a/ignite/metrics/accuracy.py b/ignite/metrics/accuracy.py
@@ -118,8 +118,8 @@ def thresholded_output_transform(output):
     def __init__(self, output_transform=lambda x: x, is_multilabel=False, device=None):
         self._num_correct = None
         self._num_examples = None
-        super(Accuracy, self).__init__(output_transform=output_transform, 
-                                       is_multilabel=is_multilabel, 
+        super(Accuracy, self).__init__(output_transform=output_transform,
+                                       is_multilabel=is_multilabel,
                                        device=device)
 
     @reinit_is_reduced

diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py
@@ -45,8 +45,8 @@ def __init__(self, num_classes, average=None, output_transform=lambda x: x, devi
 
     @reinit_is_reduced
     def reset(self):
-        self.confusion_matrix = torch.zeros(self.num_classes, self.num_classes, 
-                                            dtype=torch.float, 
+        self.confusion_matrix = torch.zeros(self.num_classes, self.num_classes,
+                                            dtype=torch.float,
                                             device=self._device)
         self._num_examples = 0
 
@@ -77,7 +77,7 @@ def _check_shape(self, output):
 
         return y_pred, y
 
-    @reinit_is_reduced 
+    @reinit_is_reduced
     def update(self, output):
         y_pred, y = self._check_shape(output)
 

diff --git a/ignite/metrics/epoch_metric.py b/ignite/metrics/epoch_metric.py
@@ -36,14 +36,12 @@ class EpochMetric(Metric):
     def __init__(self, compute_fn, output_transform=lambda x: x):
 
         if torch.distributed.is_available() and torch.distributed.is_initialized():
-            raise warnings.warn("EpochMetric class does not work in distributed setting. "
-                                "Metric's results are not reduced across the GPUs. Computed result "
-                                "corresponds to the local rank's (single GPU) result.")
+            warnings.warn("EpochMetric class does not work in distributed setting.", RuntimeWarning)
 
         if not callable(compute_fn):
             raise TypeError("Argument compute_fn should be callable.")
 
-        super(EpochMetric, self).__init__(output_transform=output_transform)
+        super(EpochMetric, self).__init__(output_transform=output_transform, device='cpu')
         self.compute_fn = compute_fn
 
     def reset(self):

diff --git a/ignite/metrics/loss.py b/ignite/metrics/loss.py
@@ -34,12 +34,12 @@ def __init__(self, loss_fn, output_transform=lambda x: x,
         self._loss_fn = loss_fn
         self._batch_size = batch_size
 
-    @reinit_is_reduced        
+    @reinit_is_reduced
     def reset(self):
         self._sum = 0
         self._num_examples = 0
 
-    @reinit_is_reduced        
+    @reinit_is_reduced
     def update(self, output):
         if len(output) == 2:
             y_pred, y = output
@@ -54,7 +54,7 @@ def update(self, output):
         N = self._batch_size(y)
         self._sum += average_loss.item() * N
         self._num_examples += N
-    
+
     @sync_all_reduce("_sum", "_num_examples")
     def compute(self):
         if self._num_examples == 0:

diff --git a/ignite/metrics/mean_pairwise_distance.py b/ignite/metrics/mean_pairwise_distance.py
@@ -10,12 +10,12 @@
 
 class MeanPairwiseDistance(Metric):
     """
-    Calculates the mean pairwise distance.
+    Calculates the mean pairwise distance: average of pairwise distances computed on provided batches.
 
     - `update` must receive output of the form `(y_pred, y)`.
     """
-    def __init__(self, p=2, eps=1e-6, output_transform=lambda x: x):
-        super(MeanPairwiseDistance, self).__init__(output_transform)
+    def __init__(self, p=2, eps=1e-6, output_transform=lambda x: x, device=None):
+        super(MeanPairwiseDistance, self).__init__(output_transform, device=device)
         self._p = p
         self._eps = eps
 

diff --git a/ignite/metrics/metric.py b/ignite/metrics/metric.py
@@ -96,7 +96,7 @@ def _sync_all_reduce(self, tensor):
         # synchronize and reduce
         torch.distributed.barrier()
         torch.distributed.all_reduce(tensor)
-        
+
         if tensor_to_number:
             return tensor.item()
         return tensor
@@ -217,11 +217,10 @@ def another_wrapper(self, *args, **kwargs):
 
 
 def reinit_is_reduced(func):
-    
+
     @wraps(func)
-    def wrapper(self, *args, **kwargs):        
+    def wrapper(self, *args, **kwargs):
         func(self, *args, **kwargs)
         self._is_reduced = False
-
-    return wrapper
 
+    return wrapper
diff --git a/ignite/metrics/metrics_lambda.py b/ignite/metrics/metrics_lambda.py
@@ -39,7 +39,7 @@ def __init__(self, f, *args, **kwargs):
         self.function = f
         self.args = args
         self.kwargs = kwargs
-        super(MetricsLambda, self).__init__()
+        super(MetricsLambda, self).__init__(device='cpu')
 
     def reset(self):
         for i in itertools.chain(self.args, self.kwargs.values()):

diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py
@@ -15,9 +15,9 @@ class _BasePrecisionRecall(_BaseClassification):
     def __init__(self, output_transform=lambda x: x, average=False, is_multilabel=False, device=None):
         if torch.distributed.is_available() and torch.distributed.is_initialized():
             if (not average) and is_multilabel:
-                raise warnings.warn("Precision/Recall metrics do not work in distributed setting when average=False "
-                                    "and is_multilabel=True. Results are not reduced across the GPUs. Computed result "
-                                    "corresponds to the local rank's (single GPU) result.")
+                warnings.warn("Precision/Recall metrics do not work in distributed setting when average=False "
+                              "and is_multilabel=True. Results are not reduced across the GPUs. Computed result "
+                              "corresponds to the local rank's (single GPU) result.", RuntimeWarning)
 
         self._average = average
         self._true_positives = None
@@ -38,7 +38,7 @@ def compute(self):
             raise NotComputableError("{} must have at least one example before"
                                      " it can be computed.".format(self.__class__.__name__))
 
-        if self._average:
+        if not (self._type == "multilabel" and not self._average):
             self._true_positives = self._sync_all_reduce(self._true_positives)
             self._positives = self._sync_all_reduce(self._positives)
 

diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py
@@ -52,11 +52,14 @@ def thresholded_output_transform(output):
             in multiclass case), otherwise, returns a tensor with the precision (for each class in multiclass case).
         is_multilabel (bool, optional) flag to use in multilabel case. By default, value is False. If True, average
             parameter should be True and the average is computed across samples, instead of classes.
+        device (str of torch.device): device specification in case of distributed computation usage.
+            In most of the cases, it should defined as "cuda:local_rank".
+
     """
 
-    def __init__(self, output_transform=lambda x: x, average=False, is_multilabel=False):
+    def __init__(self, output_transform=lambda x: x, average=False, is_multilabel=False, device=None):
         super(Recall, self).__init__(output_transform=output_transform,
-                                     average=average, is_multilabel=is_multilabel)
+                                     average=average, is_multilabel=is_multilabel, device=device)
 
     @reinit_is_reduced
     def update(self, output):

diff --git a/ignite/metrics/top_k_categorical_accuracy.py b/ignite/metrics/top_k_categorical_accuracy.py
@@ -13,8 +13,8 @@ class TopKCategoricalAccuracy(Metric):
 
     - `update` must receive output of the form `(y_pred, y)`.
     """
-    def __init__(self, k=5, output_transform=lambda x: x):
-        super(TopKCategoricalAccuracy, self).__init__(output_transform)
+    def __init__(self, k=5, output_transform=lambda x: x, device=None):
+        super(TopKCategoricalAccuracy, self).__init__(output_transform, device=device)
         self._k = k
 
     @reinit_is_reduced

diff --git a/tests/ignite/conftest.py b/tests/ignite/conftest.py
@@ -7,11 +7,16 @@
 @pytest.fixture()
 def local_rank(worker_id):
     """ use a different account in each xdist worker """
-    return int(worker_id.replace("gw", ""))
+    if "gw" in worker_id:
+        return int(worker_id.replace("gw", ""))
+    return worker_id
 
 
 @pytest.fixture()
 def distributed_context_single_node(local_rank):
+
+
+
     import os
     if "WORLD_SIZE" not in os.environ:
         os.environ["WORLD_SIZE"] = "{}".format(torch.cuda.device_count())
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,6 +6,7 @@

		import torch


		class VariableAccumulation(Metric):
		"""Single variable accumulator helper to compute (arithmetic, geometric, harmonic) average of a single variable.

Expand Down