Emit instance life cycle events

The PowerVM REST API allows a handler to be registered for events
from PowerVM. From the events it can be determined if a VM has
had a state change.  This change set registers for those events
and determines if the state change is one that should trigger
an instance life cycle event to the compute manager. The compute
manager then can sync the power state between OpenStack and
PowerVM. Normally, _sync_power_states is only called via a
periodic task which defaults to 10 minute intervals.

Change-Id: I33363d4b9dfef3fafc9125dd283660417dbf3186

nova_powervm/tests/virt/powervm/__init__.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from nova.compute import power_state
 from nova.compute import task_states
 from nova.objects import flavor
 from nova.objects import image_meta
@@ -43,6 +44,7 @@ TEST_INSTANCE = {
     'host': 'host1',
     'flavor': TEST_FLAVOR,
     'task_state': None,
+    'power_state': power_state.SHUTDOWN,
 }
 
 TEST_INST_SPAWNING = dict(TEST_INSTANCE, task_state=task_states.SPAWNING)

nova_powervm/tests/virt/powervm/fixtures.py

@@ -91,6 +91,7 @@ class PowerVMComputeDriver(fixtures.Fixture):
         # Pretend it just returned one host
         ms_http.feed.entries = [ms_http.feed.entries[0]]
         self.drv.adapter.read.return_value = ms_http
+        self.drv.session = self.drv.adapter.session
         self.drv.init_host('FakeHost')
 
     def setUp(self):

nova_powervm/tests/virt/powervm/test_driver.py

@@ -45,6 +45,7 @@ from nova_powervm.tests.virt.powervm import fixtures as fx
 from nova_powervm.virt.powervm import driver
 from nova_powervm.virt.powervm import exception as p_exc
 from nova_powervm.virt.powervm import live_migration as lpm
+from nova_powervm.virt.powervm import vm
 
 MS_HTTPRESP_FILE = "managedsystem.txt"
 MS_NAME = 'HV4'
@@ -132,6 +133,11 @@ class TestPowerVMDriver(test.TestCase):
         test_drv = driver.PowerVMDriver(fake.FakeVirtAPI())
         self.assertIsNotNone(test_drv)
 
+    def test_cleanup_host(self):
+        self.drv.cleanup_host('fake_host')
+        self.assertTrue(
+            self.drv.session.get_event_listener.return_value.shutdown.called)
+
     def test_get_volume_connector(self):
         """Tests that a volume connector can be built."""
         vol_connector = self.drv.get_volume_connector(mock.Mock())
@@ -1570,3 +1576,54 @@ class TestPowerVMDriver(test.TestCase):
         mock_bk_dev.return_value = 'info'
         self.assertEqual('info',
                          self.drv._get_block_device_info('ctx', self.inst))
+
+
+class TestNovaEventHandler(test.TestCase):
+    def setUp(self):
+        super(TestNovaEventHandler, self).setUp()
+        self.mock_driver = mock.Mock()
+        self.handler = driver.NovaEventHandler(self.mock_driver)
+
+    @mock.patch.object(vm, 'get_instance')
+    @mock.patch.object(vm, 'get_vm_qp')
+    def test_events(self, mock_qprops, mock_get_inst):
+        # Test events
+        event_data = [
+            {
+                'EventType': 'NEW_CLIENT',
+                'EventData': '',
+                'EventID': '1452692619554',
+                'EventDetail': '',
+            },
+            {
+                'EventType': 'MODIFY_URI',
+                'EventData': 'http://localhost:12080/rest/api/uom/Managed'
+                             'System/c889bf0d-9996-33ac-84c5-d16727083a77',
+                'EventID': '1452692619555',
+                'EventDetail': 'Other',
+            },
+            {
+                'EventType': 'MODIFY_URI',
+                'EventData': 'http://localhost:12080/rest/api/uom/Managed'
+                             'System/c889bf0d-9996-33ac-84c5-d16727083a77/'
+                             'LogicalPartition/794654F5-B6E9-4A51-BEC2-'
+                             'A73E41EAA938',
+                'EventID': '1452692619563',
+                'EventDetail': 'ReferenceCode,Other',
+            },
+            {
+                'EventType': 'MODIFY_URI',
+                'EventData': 'http://localhost:12080/rest/api/uom/Managed'
+                             'System/c889bf0d-9996-33ac-84c5-d16727083a77/'
+                             'LogicalPartition/794654F5-B6E9-4A51-BEC2-'
+                             'A73E41EAA938',
+                'EventID': '1452692619566',
+                'EventDetail': 'RMCState,PartitionState,Other',
+            },
+        ]
+
+        mock_qprops.return_value = pvm_bp.LPARState.RUNNING
+        mock_get_inst.return_value = powervm.TEST_INST1
+
+        self.handler.process(event_data)
+        self.assertTrue(self.mock_driver.emit_event.called)

nova_powervm/tests/virt/powervm/test_vm.py

@@ -16,13 +16,13 @@
 #
 
 import logging
-
 import mock
 
 from nova.compute import power_state
 from nova import exception
 from nova import objects
 from nova import test
+from nova.virt import event
 from pypowervm import exceptions as pvm_exc
 from pypowervm.helpers import log_helper as pvm_log
 from pypowervm.tests import test_fixtures as pvm_fx
@@ -195,6 +195,34 @@ class TestVM(test.TestCase):
 
         self.resp = lpar_http.response
 
+    def test_translate_event(self):
+        # (expected event, pvm state, power_state)
+        tests = [
+            (event.EVENT_LIFECYCLE_STARTED, "running", power_state.SHUTDOWN),
+            (None, "running", power_state.RUNNING)
+        ]
+        for t in tests:
+            self.assertEqual(t[0], vm.translate_event(t[1], t[2]))
+
+    @mock.patch.object(objects.Instance, 'get_by_uuid')
+    def test_get_instance(self, mock_get_uuid):
+        mock_get_uuid.return_value = '1111'
+        self.assertEqual('1111', vm.get_instance('ctx', 'ABC'))
+
+        mock_get_uuid.side_effect = [
+            exception.InstanceNotFound({'instance_id': 'fake_instance'}),
+            '222'
+        ]
+        self.assertEqual('222', vm.get_instance('ctx', 'ABC'))
+
+    def test_uuid_set_high_bit(self):
+        self.assertEqual(
+            vm._uuid_set_high_bit('65e7a5f0-ceb2-427d-a6d1-e47f0eb38708'),
+            'e5e7a5f0-ceb2-427d-a6d1-e47f0eb38708')
+        self.assertEqual(
+            vm._uuid_set_high_bit('f6f79d3f-eef1-4009-bfd4-172ab7e6fff4'),
+            'f6f79d3f-eef1-4009-bfd4-172ab7e6fff4')
+
     def test_instance_info(self):
 
         # Test at least one state translation

nova_powervm/virt/powervm/driver.py

@@ -26,6 +26,7 @@ from nova.objects import flavor as flavor_obj
 from nova import utils as n_utils
 from nova.virt import configdrive
 from nova.virt import driver
+from nova.virt import event
 import re
 
 from oslo_log import log as logging
@@ -41,6 +42,7 @@ from pypowervm.helpers import vios_busy as vio_hlp
 from pypowervm.tasks import memory as pvm_mem
 from pypowervm.tasks import power as pvm_pwr
 from pypowervm.tasks import vterm as pvm_vterm
+from pypowervm import util as pvm_util
 from pypowervm.wrappers import base_partition as pvm_bp
 from pypowervm.wrappers import managed_system as pvm_ms
 from pypowervm.wrappers import virtual_io_server as pvm_vios
@@ -118,11 +120,26 @@ class PowerVMDriver(driver.ComputeDriver):
 
         LOG.info(_LI("The compute driver has been initialized."))
 
+    def cleanup_host(self, host):
+        """Clean up anything that is necessary for the driver gracefully stop,
+        including ending remote sessions. This is optional.
+        """
+        # Stop listening for events
+        try:
+            self.session.get_event_listener().shutdown()
+        except Exception:
+            pass
+
+        LOG.info(_LI("The compute driver has been shutdown."))
+
     def _get_adapter(self):
         self.session = pvm_apt.Session()
         self.adapter = pvm_apt.Adapter(
             self.session, helpers=[log_hlp.log_helper,
                                    vio_hlp.vios_busy_retry_helper])
+        # Register the event handler
+        eh = NovaEventHandler(self)
+        self.session.get_event_listener().subscribe(eh)
 
     def _get_disk_adapter(self):
         conn_info = {'adapter': self.adapter, 'host_uuid': self.host_uuid,
@@ -1687,3 +1704,124 @@ class PowerVMDriver(driver.ComputeDriver):
                     return boot_conn_type
         else:
             return boot_conn_type
+
+
+class NovaEventHandler(pvm_apt.RawEventHandler):
+    """Used to receive and handle events from PowerVM."""
+    inst_actions_handled = {'PartitionState'}
+
+    def __init__(self, driver):
+        self._driver = driver
+
+    def _handle_event(self, uri, etype, details, eid):
+        """Handle an individual event.
+
+        :param uri: PowerVM event uri
+        :param etype: PowerVM event type
+        :param details: PowerVM event details
+        :param eid: PowerVM event id
+        """
+
+        # See if this uri ends with a PowerVM UUID.
+        if not pvm_util.is_instance_path(uri):
+            return
+
+        pvm_uuid = pvm_util.get_req_path_uuid(
+            uri, preserve_case=True)
+        # If a vm event and one we handle, call the inst handler.
+        if (uri.endswith('LogicalPartition/' + pvm_uuid) and
+                (self.inst_actions_handled & set(details))):
+            inst = vm.get_instance(ctx.get_admin_context(),
+                                   pvm_uuid)
+            if inst:
+                LOG.debug('Handle action "%(action)s" event for instance: '
+                          '%(inst)s' %
+                          dict(action=details, inst=inst.name))
+                self._handle_inst_event(
+                    inst, pvm_uuid, uri, etype, details, eid)
+
+    def _handle_inst_event(self, inst, pvm_uuid, uri, etype, details, eid):
+        """Handle an instance event.
+
+        This method will check if an instance event signals a change in the
+        state of the instance as known to OpenStack and if so, trigger an
+        event upward.
+
+        :param inst: the instance object.
+        :param pvm_uuid: the PowerVM uuid of the vm
+        :param uri: PowerVM event uri
+        :param etype: PowerVM event type
+        :param details: PowerVM event details
+        :param eid: PowerVM event id
+        """
+        # If the state of the vm changed see if it should be handled
+        if 'PartitionState' in details:
+            # Get the current state
+            pvm_state = vm.get_vm_qp(self._driver.adapter, pvm_uuid,
+                                     'PartitionState')
+            # See if it's really a change of state from what OpenStack knows
+            transition = vm.translate_event(pvm_state, inst.power_state)
+            if transition is not None:
+                LOG.debug('New state for instance: %s', pvm_state,
+                          instance=inst)
+                # Now create an event and sent it.
+                lce = event.LifecycleEvent(inst.uuid, transition)
+                LOG.info(_LI('Sending life cycle event for instance state '
+                             'change to: %s'), pvm_state, instance=inst)
+                self._driver.emit_event(lce)
+
+    def process(self, events):
+        """Process the event that comes back from PowerVM.
+
+        Example of event data:
+            <EventType kb="ROR" kxe="false">NEW_CLIENT</EventType>
+            <EventID kxe="false" kb="ROR">1452692619554</EventID>
+            <EventData kxe="false" kb="ROR"/>
+            <EventDetail kb="ROR" kxe="false"/>
+
+            <EventType kb="ROR" kxe="false">MODIFY_URI</EventType>
+            <EventID kxe="false" kb="ROR">1452692619557</EventID>
+            <EventData kxe="false" kb="ROR">http://localhost:12080/rest/api/
+                uom/ManagedSystem/c889bf0d-9996-33ac-84c5-d16727083a77
+            </EventData>
+            <EventDetail kb="ROR" kxe="false">Other</EventDetail>
+
+            <EventType kb="ROR" kxe="false">MODIFY_URI</EventType>
+            <EventID kxe="false" kb="ROR">1452692619566</EventID>
+            <EventData kxe="false" kb="ROR">http://localhost:12080/rest/api/
+                uom/ManagedSystem/c889bf0d-9996-33ac-84c5-d16727083a77/
+                LogicalPartition/794654F5-B6E9-4A51-BEC2-A73E41EAA938
+            </EventData>
+            <EventDetail kb="ROR" kxe="false">RMCState,PartitionState,Other
+            </EventDetail>
+
+        :param events: A sequence of event dicts that has come back from the
+                       system.
+
+                       Format:
+                       [
+                            {
+                               'EventType': <type>,
+                               'EventID': <id>,
+                               'EventData': <data>,
+                               'EventDetail': <detail>
+                            },
+                       ]
+        """
+        for pvm_event in events:
+            try:
+                # Pull all the pieces of the event.
+                uri = pvm_event['EventData']
+                etype = pvm_event['EventType']
+                details = pvm_event['EventDetail']
+                details = details.split(',') if details else []
+                eid = pvm_event['EventID']
+
+                if etype not in ['NEW_CLIENT']:
+                    LOG.debug('PowerVM Event-Action: %s URI: %s Details %s' %
+                              (etype, uri, details))
+                    self._handle_event(uri, etype, details, eid)
+            except Exception as e:
+                LOG.exception(e)
+                LOG.warning(_LW('Unable to parse event URI: %s from PowerVM.'),
+                            uri)

nova_powervm/virt/powervm/vm.py

@@ -21,6 +21,8 @@ import six
 
 from nova.compute import power_state
 from nova import exception
+from nova import objects
+from nova.virt import event
 from nova.virt import hardware
 from pypowervm import exceptions as pvm_exc
 from pypowervm.helpers import log_helper as pvm_log
@@ -47,27 +49,47 @@ LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
 POWERVM_TO_NOVA_STATE = {
-    "migrating running": power_state.RUNNING,
-    "running": power_state.RUNNING,
-    "starting": power_state.RUNNING,
+    pvm_bp.LPARState.MIGRATING_RUNNING: power_state.RUNNING,
+    pvm_bp.LPARState.RUNNING: power_state.RUNNING,
+    pvm_bp.LPARState.STARTING: power_state.RUNNING,
 
-    "migrating not active": power_state.SHUTDOWN,
-    "not activated": power_state.SHUTDOWN,
+    pvm_bp.LPARState.MIGRATING_NOT_ACTIVE: power_state.SHUTDOWN,
+    pvm_bp.LPARState.NOT_ACTIVATED: power_state.SHUTDOWN,
 
-    "hardware discovery": power_state.NOSTATE,
-    "not available": power_state.NOSTATE,
+    pvm_bp.LPARState.HARDWARE_DISCOVERY: power_state.NOSTATE,
+    pvm_bp.LPARState.NOT_AVAILBLE: power_state.NOSTATE,
     # map open firmware state to active since it can be shut down
-    "open firmware": power_state.RUNNING,
-    "resuming": power_state.NOSTATE,
-    "shutting down": power_state.NOSTATE,
-    "suspending": power_state.NOSTATE,
-    "unknown": power_state.NOSTATE,
+    pvm_bp.LPARState.OPEN_FIRMWARE: power_state.RUNNING,
+    pvm_bp.LPARState.RESUMING: power_state.NOSTATE,
+    pvm_bp.LPARState.SHUTTING_DOWN: power_state.NOSTATE,
+    pvm_bp.LPARState.SUSPENDING: power_state.NOSTATE,
+    pvm_bp.LPARState.UNKNOWN: power_state.NOSTATE,
 
-    "suspended": power_state.SUSPENDED,
+    pvm_bp.LPARState.SUSPENDED: power_state.SUSPENDED,
 
-    "error": power_state.CRASHED
+    pvm_bp.LPARState.ERROR: power_state.CRASHED
 }
 
+# Groupings of PowerVM events used when considering if a state transition
+# has taken place.
+RUNNING_EVENTS = [
+    pvm_bp.LPARState.MIGRATING_RUNNING,
+    pvm_bp.LPARState.RUNNING,
+    pvm_bp.LPARState.STARTING,
+    pvm_bp.LPARState.OPEN_FIRMWARE,
+]
+STOPPED_EVENTS = [
+    pvm_bp.LPARState.NOT_ACTIVATED,
+    pvm_bp.LPARState.ERROR,
+    pvm_bp.LPARState.UNKNOWN,
+]
+SUSPENDED_EVENTS = [
+    pvm_bp.LPARState.SUSPENDING,
+]
+RESUMING_EVENTS = [
+    pvm_bp.LPARState.RESUMING,
+]
+
 POWERVM_STARTABLE_STATE = (pvm_bp.LPARState.NOT_ACTIVATED)
 POWERVM_STOPABLE_STATE = (pvm_bp.LPARState.RUNNING, pvm_bp.LPARState.STARTING,
                           pvm_bp.LPARState.OPEN_FIRMWARE,
@@ -79,6 +101,31 @@ SECURE_RMC_VSWITCH = 'MGMTSWITCH'
 SECURE_RMC_VLAN = 4094
 
 
+def translate_event(pvm_state, pwr_state):
+    """Translate the PowerVM state and see if it has changed.
+
+    Compare the state from PowerVM to the state from OpenStack and see if
+    a life cycle event should be sent to up to OpenStack.
+
+    :param pvm_state: VM state from PowerVM
+    :param pwr_state: Instance power state from OpenStack
+    :returns: life cycle event to send.
+    """
+    trans = None
+    if pvm_state in RUNNING_EVENTS and pwr_state != power_state.RUNNING:
+        trans = event.EVENT_LIFECYCLE_STARTED
+    elif pvm_state in STOPPED_EVENTS and pwr_state != power_state.SHUTDOWN:
+        trans = event.EVENT_LIFECYCLE_STOPPED
+    elif (pvm_state in SUSPENDED_EVENTS and
+          pwr_state != power_state.SUSPENDED):
+        trans = event.EVENT_LIFECYCLE_SUSPENDED
+    elif pvm_state in RESUMING_EVENTS and pwr_state != power_state.RUNNING:
+        trans = event.EVENT_LIFECYCLE_RESUMED
+
+    LOG.debug('Transistion to %s' % trans)
+    return trans
+
+
 def _translate_vm_state(pvm_state):
     """Find the current state of the lpar and convert it to
     the appropriate nova.compute.power_state
@@ -607,6 +654,45 @@ def get_pvm_uuid(instance):
     return pvm_uuid.convert_uuid_to_pvm(instance.uuid).upper()
 
 
+def _uuid_set_high_bit(pvm_uuid):
+    """Turns on the high bit of a uuid
+
+    PowerVM uuids always set the byte 0, bit 0 to 0.
+    So to convert it to an OpenStack uuid we may have to set the high bit.
+
+    :param uuid: A PowerVM compliant uuid
+    :returns: A standard format uuid string
+    """
+    return "%x%s" % (int(pvm_uuid[0], 16) | 8, pvm_uuid[1:])
+
+
+def get_instance(context, pvm_uuid):
+    """Get an instance, if there is one, that corresponds to the PVM UUID
+
+    Not finding the instance can be a pretty normal case when handling events.
+    Don't log exceptions for those cases.
+
+    :param pvm_uuid: PowerVM UUID
+    :return: OpenStack instance or None
+    """
+    uuid = pvm_uuid.lower()
+
+    def get_inst():
+        try:
+            return objects.Instance.get_by_uuid(context, uuid)
+        except exception.InstanceNotFound:
+            return objects.Instance.get_by_uuid(context,
+                                                _uuid_set_high_bit(uuid))
+
+    try:
+        return get_inst()
+    except exception.InstanceNotFound:
+        pass
+    except Exception as e:
+        LOG.debug('PowerVM UUID not found. %s', e)
+    return None
+
+
 def get_cnas(adapter, instance, host_uuid):
     """Returns the current CNAs on the instance.