Merge pull request #3447 from heplesser/mpitest_error_reporting

Handle CalledProcessError better

Author: heplesser

pynest/nest/lib/hl_api_types.py

@@ -868,8 +868,13 @@ def get(self, keys=None, output=""):
         # This avoids problems with invalid SynapseCollections.
         # See also #3100.
         if self.__len__() == 0 or GetKernelStatus("network_size") == 0:
-            # Return empty tuple if get is called with an argument
-            return {} if keys is None else ()
+            if pandas_output:
+                return pandas.DataFrame()
+            elif output == "json":
+                return to_json({})
+            else:
+                # Return empty tuple if get is called with an argument
+                return {} if keys is None else ()
 
         if keys is None:
             cmd = "GetStatus"

testsuite/pytests/sli2py_mpi/mpi_test_wrapper.py

@@ -183,12 +183,29 @@ def wrapper(*args, **kwargs):
 
             res = {}
             for procs in self._procs_lst:
-                res[procs] = subprocess.run(
-                    ["mpirun", "-np", str(procs), "--oversubscribe", "python", self.RUNNER],
-                    check=True,
-                    cwd=tmpdirpath,
-                    capture_output=self._debug,
-                )
+                try:
+                    command = ["mpirun", "-np", str(procs), "--oversubscribe", "python", self.RUNNER]
+                    res[procs] = subprocess.run(
+                        command,
+                        check=True,
+                        cwd=tmpdirpath,
+                        capture_output=True,  # always capture, in case an error is thrown
+                    )
+                except subprocess.CalledProcessError as err:
+                    print("\n")
+                    print("-" * 50)
+                    print(f"Running failed for: {command}")
+                    print(f"tmpdir            : {tmpdir.name}")
+                    print("-" * 50)
+                    print("STDOUT")
+                    print("-" * 50)
+                    print(err.stdout.decode("UTF-8"))
+                    print("-" * 50)
+                    print("STDERR")
+                    print("-" * 50)
+                    print(err.stderr.decode("UTF-8"))
+                    print("-" * 50)
+                    raise err
 
             if self._debug:
                 print("\n\n")
@@ -210,10 +227,17 @@ def _collect_result_by_label(self, tmpdirpath, label):
         except StopIteration:
             return None  # no data for this label
 
-        return {
-            n_procs: [pd.read_csv(f, sep="\t", comment="#") for f in tmpdirpath.glob(label.format(n_procs, "*"))]
-            for n_procs in self._procs_lst
-        }
+        res = {}
+        for n_procs in self._procs_lst:
+            data = []
+            for f in tmpdirpath.glob(label.format(n_procs, "*")):
+                try:
+                    data.append(pd.read_csv(f, sep="\t", comment="#"))
+                except pd.errors.EmptyDataError:
+                    pass
+            res[n_procs] = data
+
+        return res
 
     def collect_results(self, tmpdirpath):
         """

testsuite/pytests/sli2py_mpi/test_self_get_conns_with_empty_ranks.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# test_self_get_conns_with_empty_ranks.py
+#
+# This file is part of NEST.
+#
+# Copyright (C) 2004 The NEST Initiative
+#
+# NEST is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# NEST is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with NEST.  If not, see <http://www.gnu.org/licenses/>.
+
+import numpy as np
+import pandas
+import pytest
+from mpi_test_wrapper import MPITestAssertEqual
+
+
+# Parametrization over the number of nodes here only to show hat it works
+@MPITestAssertEqual([1, 2, 4], debug=False)
+def test_self_get_conns_with_empty_ranks():
+    """
+    Selftest: Confirm that connections can be gathered correctly even if some ranks have no neurons.
+    """
+
+    import nest
+    import pandas as pd
+
+    nrns = nest.Create("parrot_neuron", n=2)
+    nest.Connect(nrns, nrns)
+
+    conns = nest.GetConnections().get(output="pandas").drop(labels=["target_thread", "port"], axis=1, errors="ignore")
+    conns.to_csv(OTHER_LABEL.format(nest.num_processes, nest.Rank()), index=False)  # noqa: F821