Handle depth errors in get_running_builds

There are cases where asking for depth>0 node_info will 500. This breaks
listing jobs because we raise and stop looking for running jobs. Handle
this by checking if it is a 500 error at depth=2 and if so check
depth=0. If that returns successfully treat it as a broken slave that
isn't running any jobs.

If depth=0 returns an error things are probably much more broken and
should be looked into.

Change-Id: Ieac15a0fe2a47ec3dae51db96ad2fe40992c353a

jenkins/__init__.py

@@ -994,7 +994,16 @@ class Jenkins(object):
                 node_name = '(master)'
             else:
                 node_name = node['name']
-            info = self.get_node_info(node_name, depth=2)
+            try:
+                info = self.get_node_info(node_name, depth=2)
+            except JenkinsException as e:
+                # Jenkins may 500 on depth >0. If the node info comes back
+                # at depth 0 treat it as a node not running any jobs.
+                if ('[500]' in str(e) and
+                        self.get_node_info(node_name, depth=0)):
+                    continue
+                else:
+                    raise
             for executor in info['executors']:
                 executable = executor['currentExecutable']
                 if executable:

tests/test_build.py

@@ -396,3 +396,24 @@ class JenkinsListRunningBuildsTest(JenkinsTestBase):
         node_info_mock.return_value = node_info_to_return
         builds = self.j.get_running_builds()
         self.assertEqual([], builds)
+
+    @patch.object(jenkins.Jenkins, 'get_node_info')
+    @patch.object(jenkins.Jenkins, 'get_nodes')
+    def test_broken_slave(self, nodes_mock, node_info_mock):
+        nodes_to_return = [{
+            'name': "foo-slave", 'offline': False
+        }]
+        nodes_mock.return_value = nodes_to_return
+
+        def side_effect(*args, **kwargs):
+            if 'depth' in kwargs and kwargs['depth'] > 0:
+                raise jenkins.JenkinsException(
+                    "Error in request. Possibly authentication failed"
+                    "[500]: Server Error")
+            else:
+                return {"success": True}
+
+        node_info_mock.side_effect = side_effect
+        builds = self.j.get_running_builds()
+        # Should treat the slave as not running any builds
+        self.assertEqual([], builds)