Cache server status

In a large environment with a lot of bmcs running, just the periodic
status checks from something like Ironic can generate a lot of api
traffic on the host cloud.  Since nothing should be changing the
status of the instances out from under us, we can cache the status
and only look it up when we know it may have changed.  This should
reduce the number of api calls significantly.

For reference, it looks like Ironic checks power status once per
minute by default.  In an environment with 5 baremetal nodes per
baremetal stack and 50 total stacks (not a particularly huge
environment), this can generate 250 server gets per minute, which
seems to have the knock-on effect of generating a lot of Neutron
traffic too.  My assumption is that Nova is retrieving network
details for the server each time the server is retrieved, and in a
net-iso stack this involves a lot of networks and ports.

openstack_virtual_baremetal/openstackbmc.py

@@ -39,6 +39,8 @@ class OpenStackBmc(bmc.Bmc):
         self.novaclient = novaclient.Client(2, user, password,
                                             tenant, auth_url)
         self.instance = None
+        self.cached_status = None
+        self.target_status = None
         # At times the bmc service is started before important things like
         # networking have fully initialized.  Keep trying to find the
         # instance indefinitely, since there's no point in continuing if
@@ -91,7 +93,9 @@ class OpenStackBmc(bmc.Bmc):
         sys.exit(0)
 
     def _instance_active(self):
-        return self.novaclient.servers.get(self.instance).status == 'ACTIVE'
+        if self.cached_status is None or self.cached_status != self.target_status:
+            self.cached_status = self.novaclient.servers.get(self.instance).status
+        return self.cached_status == 'ACTIVE'
 
     def get_power_state(self):
         self.log('Getting power state for %s' % self.instance)
@@ -99,6 +103,7 @@ class OpenStackBmc(bmc.Bmc):
 
     def power_off(self):
         # this should be power down without waiting for clean shutdown
+        self.target_status = 'SHUTOFF'
         if self._instance_active():
             try:
                 self.novaclient.servers.stop(self.instance)
@@ -113,6 +118,7 @@ class OpenStackBmc(bmc.Bmc):
             return 0xd5
 
     def power_on(self):
+        self.target_status = 'ACTIVE'
         if not self._instance_active():
             try:
                 self.novaclient.servers.start(self.instance)
@@ -131,6 +137,7 @@ class OpenStackBmc(bmc.Bmc):
 
     def power_shutdown(self):
         # should attempt a clean shutdown
+        self.target_status = 'ACTIVE'
         self.novaclient.servers.stop(self.instance)
         self.log('Politely shut down %s' % self.instance)
 

openstack_virtual_baremetal/tests/test_openstackbmc.py

@@ -103,6 +103,8 @@ class TestOpenStackBmc(unittest.TestCase):
                                              )
         self.bmc.novaclient = self.mock_client
         self.bmc.instance = 'abc-123'
+        self.bmc.cached_status = None
+        self.bmc.target_status = None
 
     def test_find_instance(self, mock_nova, mock_log, mock_init):
         self._create_bmc(mock_nova)
@@ -194,6 +196,22 @@ class TestOpenStackBmc(unittest.TestCase):
         self.mock_client.servers.get.return_value = mock_server
         self.assertFalse(self.bmc._instance_active())
 
+    def test_instance_active_mismatch(self, mock_nova, mock_log, mock_init):
+        self._create_bmc(mock_nova)
+        mock_server = mock.Mock()
+        mock_server.status = 'ACTIVE'
+        self.mock_client.servers.get.return_value = mock_server
+        self.bmc.target_status = 'ACTIVE'
+        self.bmc.cached_status = 'SHUTOFF'
+        self.assertTrue(self.bmc._instance_active())
+
+    def test_instance_active_cached(self, mock_nova, mock_log, mock_init):
+        self._create_bmc(mock_nova)
+        self.bmc.target_status = 'ACTIVE'
+        self.bmc.cached_status = 'ACTIVE'
+        self.assertTrue(self.bmc._instance_active())
+        self.assertFalse(self.mock_client.servers.get.called)
+
     @mock.patch('openstack_virtual_baremetal.openstackbmc.OpenStackBmc.'
                 '_instance_active')
     def test_get_power_state(self, mock_active, mock_nova, mock_log,
@@ -209,6 +227,7 @@ class TestOpenStackBmc(unittest.TestCase):
         mock_active.return_value = True
         self.bmc.power_off()
         self.mock_client.servers.stop.assert_called_once_with('abc-123')
+        self.assertEqual('SHUTOFF', self.bmc.target_status)
 
     @mock.patch('openstack_virtual_baremetal.openstackbmc.OpenStackBmc.'
                 '_instance_active')
@@ -239,6 +258,7 @@ class TestOpenStackBmc(unittest.TestCase):
         mock_active.return_value = False
         self.bmc.power_on()
         self.mock_client.servers.start.assert_called_once_with('abc-123')
+        self.assertEqual('ACTIVE', self.bmc.target_status)
 
     @mock.patch('openstack_virtual_baremetal.openstackbmc.OpenStackBmc.'
                 '_instance_active')