From 97b36614638b38c858ae2a1747f28f1ecee201f9 Mon Sep 17 00:00:00 2001 From: "Kyle L. Henderson" Date: Tue, 26 Jan 2016 14:31:20 -0600 Subject: [PATCH] Emit instance life cycle events The PowerVM REST API allows a handler to be registered for events from PowerVM. From the events it can be determined if a VM has had a state change. This change set registers for those events and determines if the state change is one that should trigger an instance life cycle event to the compute manager. The compute manager then can sync the power state between OpenStack and PowerVM. Normally, _sync_power_states is only called via a periodic task which defaults to 10 minute intervals. Change-Id: I33363d4b9dfef3fafc9125dd283660417dbf3186 --- nova_powervm/tests/virt/powervm/__init__.py | 2 + nova_powervm/tests/virt/powervm/fixtures.py | 1 + .../tests/virt/powervm/test_driver.py | 57 ++++++++ nova_powervm/tests/virt/powervm/test_vm.py | 30 +++- nova_powervm/virt/powervm/driver.py | 138 ++++++++++++++++++ nova_powervm/virt/powervm/vm.py | 114 +++++++++++++-- 6 files changed, 327 insertions(+), 15 deletions(-) diff --git a/nova_powervm/tests/virt/powervm/__init__.py b/nova_powervm/tests/virt/powervm/__init__.py index 5502b47f..d4c57282 100644 --- a/nova_powervm/tests/virt/powervm/__init__.py +++ b/nova_powervm/tests/virt/powervm/__init__.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nova.compute import power_state from nova.compute import task_states from nova.objects import flavor from nova.objects import image_meta @@ -43,6 +44,7 @@ TEST_INSTANCE = { 'host': 'host1', 'flavor': TEST_FLAVOR, 'task_state': None, + 'power_state': power_state.SHUTDOWN, } TEST_INST_SPAWNING = dict(TEST_INSTANCE, task_state=task_states.SPAWNING) diff --git a/nova_powervm/tests/virt/powervm/fixtures.py b/nova_powervm/tests/virt/powervm/fixtures.py index 81633de1..805eefaa 100644 --- a/nova_powervm/tests/virt/powervm/fixtures.py +++ b/nova_powervm/tests/virt/powervm/fixtures.py @@ -91,6 +91,7 @@ class PowerVMComputeDriver(fixtures.Fixture): # Pretend it just returned one host ms_http.feed.entries = [ms_http.feed.entries[0]] self.drv.adapter.read.return_value = ms_http + self.drv.session = self.drv.adapter.session self.drv.init_host('FakeHost') def setUp(self): diff --git a/nova_powervm/tests/virt/powervm/test_driver.py b/nova_powervm/tests/virt/powervm/test_driver.py index 2911035a..ef16988e 100644 --- a/nova_powervm/tests/virt/powervm/test_driver.py +++ b/nova_powervm/tests/virt/powervm/test_driver.py @@ -45,6 +45,7 @@ from nova_powervm.tests.virt.powervm import fixtures as fx from nova_powervm.virt.powervm import driver from nova_powervm.virt.powervm import exception as p_exc from nova_powervm.virt.powervm import live_migration as lpm +from nova_powervm.virt.powervm import vm MS_HTTPRESP_FILE = "managedsystem.txt" MS_NAME = 'HV4' @@ -132,6 +133,11 @@ class TestPowerVMDriver(test.TestCase): test_drv = driver.PowerVMDriver(fake.FakeVirtAPI()) self.assertIsNotNone(test_drv) + def test_cleanup_host(self): + self.drv.cleanup_host('fake_host') + self.assertTrue( + self.drv.session.get_event_listener.return_value.shutdown.called) + def test_get_volume_connector(self): """Tests that a volume connector can be built.""" vol_connector = self.drv.get_volume_connector(mock.Mock()) @@ -1570,3 +1576,54 @@ class TestPowerVMDriver(test.TestCase): mock_bk_dev.return_value = 'info' self.assertEqual('info', self.drv._get_block_device_info('ctx', self.inst)) + + +class TestNovaEventHandler(test.TestCase): + def setUp(self): + super(TestNovaEventHandler, self).setUp() + self.mock_driver = mock.Mock() + self.handler = driver.NovaEventHandler(self.mock_driver) + + @mock.patch.object(vm, 'get_instance') + @mock.patch.object(vm, 'get_vm_qp') + def test_events(self, mock_qprops, mock_get_inst): + # Test events + event_data = [ + { + 'EventType': 'NEW_CLIENT', + 'EventData': '', + 'EventID': '1452692619554', + 'EventDetail': '', + }, + { + 'EventType': 'MODIFY_URI', + 'EventData': 'http://localhost:12080/rest/api/uom/Managed' + 'System/c889bf0d-9996-33ac-84c5-d16727083a77', + 'EventID': '1452692619555', + 'EventDetail': 'Other', + }, + { + 'EventType': 'MODIFY_URI', + 'EventData': 'http://localhost:12080/rest/api/uom/Managed' + 'System/c889bf0d-9996-33ac-84c5-d16727083a77/' + 'LogicalPartition/794654F5-B6E9-4A51-BEC2-' + 'A73E41EAA938', + 'EventID': '1452692619563', + 'EventDetail': 'ReferenceCode,Other', + }, + { + 'EventType': 'MODIFY_URI', + 'EventData': 'http://localhost:12080/rest/api/uom/Managed' + 'System/c889bf0d-9996-33ac-84c5-d16727083a77/' + 'LogicalPartition/794654F5-B6E9-4A51-BEC2-' + 'A73E41EAA938', + 'EventID': '1452692619566', + 'EventDetail': 'RMCState,PartitionState,Other', + }, + ] + + mock_qprops.return_value = pvm_bp.LPARState.RUNNING + mock_get_inst.return_value = powervm.TEST_INST1 + + self.handler.process(event_data) + self.assertTrue(self.mock_driver.emit_event.called) diff --git a/nova_powervm/tests/virt/powervm/test_vm.py b/nova_powervm/tests/virt/powervm/test_vm.py index 9805d9e6..b0ce31d1 100644 --- a/nova_powervm/tests/virt/powervm/test_vm.py +++ b/nova_powervm/tests/virt/powervm/test_vm.py @@ -16,13 +16,13 @@ # import logging - import mock from nova.compute import power_state from nova import exception from nova import objects from nova import test +from nova.virt import event from pypowervm import exceptions as pvm_exc from pypowervm.helpers import log_helper as pvm_log from pypowervm.tests import test_fixtures as pvm_fx @@ -195,6 +195,34 @@ class TestVM(test.TestCase): self.resp = lpar_http.response + def test_translate_event(self): + # (expected event, pvm state, power_state) + tests = [ + (event.EVENT_LIFECYCLE_STARTED, "running", power_state.SHUTDOWN), + (None, "running", power_state.RUNNING) + ] + for t in tests: + self.assertEqual(t[0], vm.translate_event(t[1], t[2])) + + @mock.patch.object(objects.Instance, 'get_by_uuid') + def test_get_instance(self, mock_get_uuid): + mock_get_uuid.return_value = '1111' + self.assertEqual('1111', vm.get_instance('ctx', 'ABC')) + + mock_get_uuid.side_effect = [ + exception.InstanceNotFound({'instance_id': 'fake_instance'}), + '222' + ] + self.assertEqual('222', vm.get_instance('ctx', 'ABC')) + + def test_uuid_set_high_bit(self): + self.assertEqual( + vm._uuid_set_high_bit('65e7a5f0-ceb2-427d-a6d1-e47f0eb38708'), + 'e5e7a5f0-ceb2-427d-a6d1-e47f0eb38708') + self.assertEqual( + vm._uuid_set_high_bit('f6f79d3f-eef1-4009-bfd4-172ab7e6fff4'), + 'f6f79d3f-eef1-4009-bfd4-172ab7e6fff4') + def test_instance_info(self): # Test at least one state translation diff --git a/nova_powervm/virt/powervm/driver.py b/nova_powervm/virt/powervm/driver.py index 537aa48d..207949e9 100644 --- a/nova_powervm/virt/powervm/driver.py +++ b/nova_powervm/virt/powervm/driver.py @@ -26,6 +26,7 @@ from nova.objects import flavor as flavor_obj from nova import utils as n_utils from nova.virt import configdrive from nova.virt import driver +from nova.virt import event import re from oslo_log import log as logging @@ -41,6 +42,7 @@ from pypowervm.helpers import vios_busy as vio_hlp from pypowervm.tasks import memory as pvm_mem from pypowervm.tasks import power as pvm_pwr from pypowervm.tasks import vterm as pvm_vterm +from pypowervm import util as pvm_util from pypowervm.wrappers import base_partition as pvm_bp from pypowervm.wrappers import managed_system as pvm_ms from pypowervm.wrappers import virtual_io_server as pvm_vios @@ -118,11 +120,26 @@ class PowerVMDriver(driver.ComputeDriver): LOG.info(_LI("The compute driver has been initialized.")) + def cleanup_host(self, host): + """Clean up anything that is necessary for the driver gracefully stop, + including ending remote sessions. This is optional. + """ + # Stop listening for events + try: + self.session.get_event_listener().shutdown() + except Exception: + pass + + LOG.info(_LI("The compute driver has been shutdown.")) + def _get_adapter(self): self.session = pvm_apt.Session() self.adapter = pvm_apt.Adapter( self.session, helpers=[log_hlp.log_helper, vio_hlp.vios_busy_retry_helper]) + # Register the event handler + eh = NovaEventHandler(self) + self.session.get_event_listener().subscribe(eh) def _get_disk_adapter(self): conn_info = {'adapter': self.adapter, 'host_uuid': self.host_uuid, @@ -1687,3 +1704,124 @@ class PowerVMDriver(driver.ComputeDriver): return boot_conn_type else: return boot_conn_type + + +class NovaEventHandler(pvm_apt.RawEventHandler): + """Used to receive and handle events from PowerVM.""" + inst_actions_handled = {'PartitionState'} + + def __init__(self, driver): + self._driver = driver + + def _handle_event(self, uri, etype, details, eid): + """Handle an individual event. + + :param uri: PowerVM event uri + :param etype: PowerVM event type + :param details: PowerVM event details + :param eid: PowerVM event id + """ + + # See if this uri ends with a PowerVM UUID. + if not pvm_util.is_instance_path(uri): + return + + pvm_uuid = pvm_util.get_req_path_uuid( + uri, preserve_case=True) + # If a vm event and one we handle, call the inst handler. + if (uri.endswith('LogicalPartition/' + pvm_uuid) and + (self.inst_actions_handled & set(details))): + inst = vm.get_instance(ctx.get_admin_context(), + pvm_uuid) + if inst: + LOG.debug('Handle action "%(action)s" event for instance: ' + '%(inst)s' % + dict(action=details, inst=inst.name)) + self._handle_inst_event( + inst, pvm_uuid, uri, etype, details, eid) + + def _handle_inst_event(self, inst, pvm_uuid, uri, etype, details, eid): + """Handle an instance event. + + This method will check if an instance event signals a change in the + state of the instance as known to OpenStack and if so, trigger an + event upward. + + :param inst: the instance object. + :param pvm_uuid: the PowerVM uuid of the vm + :param uri: PowerVM event uri + :param etype: PowerVM event type + :param details: PowerVM event details + :param eid: PowerVM event id + """ + # If the state of the vm changed see if it should be handled + if 'PartitionState' in details: + # Get the current state + pvm_state = vm.get_vm_qp(self._driver.adapter, pvm_uuid, + 'PartitionState') + # See if it's really a change of state from what OpenStack knows + transition = vm.translate_event(pvm_state, inst.power_state) + if transition is not None: + LOG.debug('New state for instance: %s', pvm_state, + instance=inst) + # Now create an event and sent it. + lce = event.LifecycleEvent(inst.uuid, transition) + LOG.info(_LI('Sending life cycle event for instance state ' + 'change to: %s'), pvm_state, instance=inst) + self._driver.emit_event(lce) + + def process(self, events): + """Process the event that comes back from PowerVM. + + Example of event data: + NEW_CLIENT + 1452692619554 + + + + MODIFY_URI + 1452692619557 + http://localhost:12080/rest/api/ + uom/ManagedSystem/c889bf0d-9996-33ac-84c5-d16727083a77 + + Other + + MODIFY_URI + 1452692619566 + http://localhost:12080/rest/api/ + uom/ManagedSystem/c889bf0d-9996-33ac-84c5-d16727083a77/ + LogicalPartition/794654F5-B6E9-4A51-BEC2-A73E41EAA938 + + RMCState,PartitionState,Other + + + :param events: A sequence of event dicts that has come back from the + system. + + Format: + [ + { + 'EventType': , + 'EventID': , + 'EventData': , + 'EventDetail': + }, + ] + """ + for pvm_event in events: + try: + # Pull all the pieces of the event. + uri = pvm_event['EventData'] + etype = pvm_event['EventType'] + details = pvm_event['EventDetail'] + details = details.split(',') if details else [] + eid = pvm_event['EventID'] + + if etype not in ['NEW_CLIENT']: + LOG.debug('PowerVM Event-Action: %s URI: %s Details %s' % + (etype, uri, details)) + self._handle_event(uri, etype, details, eid) + except Exception as e: + LOG.exception(e) + LOG.warning(_LW('Unable to parse event URI: %s from PowerVM.'), + uri) diff --git a/nova_powervm/virt/powervm/vm.py b/nova_powervm/virt/powervm/vm.py index 8d6b7e91..1c15cd71 100644 --- a/nova_powervm/virt/powervm/vm.py +++ b/nova_powervm/virt/powervm/vm.py @@ -21,6 +21,8 @@ import six from nova.compute import power_state from nova import exception +from nova import objects +from nova.virt import event from nova.virt import hardware from pypowervm import exceptions as pvm_exc from pypowervm.helpers import log_helper as pvm_log @@ -47,27 +49,47 @@ LOG = logging.getLogger(__name__) CONF = cfg.CONF POWERVM_TO_NOVA_STATE = { - "migrating running": power_state.RUNNING, - "running": power_state.RUNNING, - "starting": power_state.RUNNING, + pvm_bp.LPARState.MIGRATING_RUNNING: power_state.RUNNING, + pvm_bp.LPARState.RUNNING: power_state.RUNNING, + pvm_bp.LPARState.STARTING: power_state.RUNNING, - "migrating not active": power_state.SHUTDOWN, - "not activated": power_state.SHUTDOWN, + pvm_bp.LPARState.MIGRATING_NOT_ACTIVE: power_state.SHUTDOWN, + pvm_bp.LPARState.NOT_ACTIVATED: power_state.SHUTDOWN, - "hardware discovery": power_state.NOSTATE, - "not available": power_state.NOSTATE, + pvm_bp.LPARState.HARDWARE_DISCOVERY: power_state.NOSTATE, + pvm_bp.LPARState.NOT_AVAILBLE: power_state.NOSTATE, # map open firmware state to active since it can be shut down - "open firmware": power_state.RUNNING, - "resuming": power_state.NOSTATE, - "shutting down": power_state.NOSTATE, - "suspending": power_state.NOSTATE, - "unknown": power_state.NOSTATE, + pvm_bp.LPARState.OPEN_FIRMWARE: power_state.RUNNING, + pvm_bp.LPARState.RESUMING: power_state.NOSTATE, + pvm_bp.LPARState.SHUTTING_DOWN: power_state.NOSTATE, + pvm_bp.LPARState.SUSPENDING: power_state.NOSTATE, + pvm_bp.LPARState.UNKNOWN: power_state.NOSTATE, - "suspended": power_state.SUSPENDED, + pvm_bp.LPARState.SUSPENDED: power_state.SUSPENDED, - "error": power_state.CRASHED + pvm_bp.LPARState.ERROR: power_state.CRASHED } +# Groupings of PowerVM events used when considering if a state transition +# has taken place. +RUNNING_EVENTS = [ + pvm_bp.LPARState.MIGRATING_RUNNING, + pvm_bp.LPARState.RUNNING, + pvm_bp.LPARState.STARTING, + pvm_bp.LPARState.OPEN_FIRMWARE, +] +STOPPED_EVENTS = [ + pvm_bp.LPARState.NOT_ACTIVATED, + pvm_bp.LPARState.ERROR, + pvm_bp.LPARState.UNKNOWN, +] +SUSPENDED_EVENTS = [ + pvm_bp.LPARState.SUSPENDING, +] +RESUMING_EVENTS = [ + pvm_bp.LPARState.RESUMING, +] + POWERVM_STARTABLE_STATE = (pvm_bp.LPARState.NOT_ACTIVATED) POWERVM_STOPABLE_STATE = (pvm_bp.LPARState.RUNNING, pvm_bp.LPARState.STARTING, pvm_bp.LPARState.OPEN_FIRMWARE, @@ -79,6 +101,31 @@ SECURE_RMC_VSWITCH = 'MGMTSWITCH' SECURE_RMC_VLAN = 4094 +def translate_event(pvm_state, pwr_state): + """Translate the PowerVM state and see if it has changed. + + Compare the state from PowerVM to the state from OpenStack and see if + a life cycle event should be sent to up to OpenStack. + + :param pvm_state: VM state from PowerVM + :param pwr_state: Instance power state from OpenStack + :returns: life cycle event to send. + """ + trans = None + if pvm_state in RUNNING_EVENTS and pwr_state != power_state.RUNNING: + trans = event.EVENT_LIFECYCLE_STARTED + elif pvm_state in STOPPED_EVENTS and pwr_state != power_state.SHUTDOWN: + trans = event.EVENT_LIFECYCLE_STOPPED + elif (pvm_state in SUSPENDED_EVENTS and + pwr_state != power_state.SUSPENDED): + trans = event.EVENT_LIFECYCLE_SUSPENDED + elif pvm_state in RESUMING_EVENTS and pwr_state != power_state.RUNNING: + trans = event.EVENT_LIFECYCLE_RESUMED + + LOG.debug('Transistion to %s' % trans) + return trans + + def _translate_vm_state(pvm_state): """Find the current state of the lpar and convert it to the appropriate nova.compute.power_state @@ -607,6 +654,45 @@ def get_pvm_uuid(instance): return pvm_uuid.convert_uuid_to_pvm(instance.uuid).upper() +def _uuid_set_high_bit(pvm_uuid): + """Turns on the high bit of a uuid + + PowerVM uuids always set the byte 0, bit 0 to 0. + So to convert it to an OpenStack uuid we may have to set the high bit. + + :param uuid: A PowerVM compliant uuid + :returns: A standard format uuid string + """ + return "%x%s" % (int(pvm_uuid[0], 16) | 8, pvm_uuid[1:]) + + +def get_instance(context, pvm_uuid): + """Get an instance, if there is one, that corresponds to the PVM UUID + + Not finding the instance can be a pretty normal case when handling events. + Don't log exceptions for those cases. + + :param pvm_uuid: PowerVM UUID + :return: OpenStack instance or None + """ + uuid = pvm_uuid.lower() + + def get_inst(): + try: + return objects.Instance.get_by_uuid(context, uuid) + except exception.InstanceNotFound: + return objects.Instance.get_by_uuid(context, + _uuid_set_high_bit(uuid)) + + try: + return get_inst() + except exception.InstanceNotFound: + pass + except Exception as e: + LOG.debug('PowerVM UUID not found. %s', e) + return None + + def get_cnas(adapter, instance, host_uuid): """Returns the current CNAs on the instance.