InstanceFailure: Ignore notifications for certain events
For instance failure recovery Masakari doesn't check for the event from payload received in notification, as a result it executes workflow for all the events received. When a instance is stopped (normal operation), masakari-instancemonitor sends notifications related to following two events one after another to masakari-api service. A) event: LIFECYCLE detail: STOPPED_SHUTDOWN B) event: LIFECYCLE detail: SHUTDOWN_FINISHED When masakari-engine processes the first notification, it checks for vmstate of the instance, if it’s 'stopped', it won't stop the vm. But if it's not stopped it will attempt to stop it. Then again, libvirt will send the above two events to masakari-instancemonitor and masakari-instancemonitor will send related notifications to masakari-api and this way it will go into the recursion. To solve this issue we can only process the below mentioned events at masakari side which actually requires actions to be taken: A) 'event': 'LIFECYCLE', 'vir_domain_event': 'STOPPED_FAILED' B) 'event': 'IO_ERROR', 'vir_domain_event': 'IO_ERROR_REPORT' So removed handling of all other events. For all other events masakari will simply log the message at INFO level and mark the notification status as 'ignored'. Closes-Bug: #1662994 Change-Id: If1113a599894cc7bab32b9656b6c42de5e5d7a54
This commit is contained in:
parent
6db17bc33a
commit
74dff4b446
34
masakari/engine/instance_events.py
Normal file
34
masakari/engine/instance_events.py
Normal file
@ -0,0 +1,34 @@
|
||||
# Copyright 2017 NTT DATA
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
""" VM libvirt events
|
||||
|
||||
These are the events which needs to be processed by masakari in case of
|
||||
instance recovery failure.
|
||||
"""
|
||||
|
||||
INSTANCE_EVENTS = {
|
||||
# Add more events and vir_domain_events here.
|
||||
'LIFECYCLE': ['STOPPED_FAILED'],
|
||||
'IO_ERROR': ['IO_ERROR_REPORT']
|
||||
}
|
||||
|
||||
|
||||
def is_valid_event(payload):
|
||||
vir_domain_event_list = INSTANCE_EVENTS.get(payload.get('event'))
|
||||
if vir_domain_event_list and payload.get(
|
||||
'vir_domain_event') in vir_domain_event_list:
|
||||
return True
|
||||
return False
|
@ -29,6 +29,7 @@ from oslo_utils import timeutils
|
||||
|
||||
import masakari.conf
|
||||
from masakari.engine import driver
|
||||
from masakari.engine import instance_events as virt_events
|
||||
from masakari import exception
|
||||
from masakari.i18n import _LE, _LI, _LW
|
||||
from masakari import manager
|
||||
@ -100,6 +101,13 @@ class MasakariManager(manager.Manager):
|
||||
return notification_status
|
||||
|
||||
def _handle_notification_type_instance(self, context, notification):
|
||||
if not virt_events.is_valid_event(notification.payload):
|
||||
LOG.info(_LI("Notification '%(uuid)s' received with payload "
|
||||
"%(payload)s is ignored."), {
|
||||
"uuid": notification.notification_uuid,
|
||||
"payload": notification.payload})
|
||||
return fields.NotificationStatus.IGNORED
|
||||
|
||||
notification_status = fields.NotificationStatus.FINISHED
|
||||
try:
|
||||
self.driver.execute_instance_failure(
|
||||
|
@ -45,8 +45,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
|
||||
def _get_vm_type_notification(self):
|
||||
return fakes.create_fake_notification(
|
||||
type="VM", id=1, payload={
|
||||
'event': 'stopped', 'instance_uuid': uuidsentinel.fake_ins,
|
||||
'vir_domain_event': 'fake_event'
|
||||
'event': 'LIFECYCLE', 'instance_uuid': uuidsentinel.fake_ins,
|
||||
'vir_domain_event': 'STOPPED_FAILED'
|
||||
},
|
||||
source_host_uuid=uuidsentinel.fake_host,
|
||||
generated_time=NOW, status="new",
|
||||
@ -100,6 +100,22 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
|
||||
self.context, notification.payload.get('instance_uuid'),
|
||||
notification.notification_uuid)
|
||||
|
||||
@mock.patch.object(notification_obj.Notification, "save")
|
||||
def test_process_notification_type_vm_error_event_unmatched(
|
||||
self, mock_save):
|
||||
notification = fakes.create_fake_notification(
|
||||
type="VM", id=1, payload={
|
||||
'event': 'fake_event', 'instance_uuid': uuidsentinel.fake_ins,
|
||||
'vir_domain_event': 'fake_vir_domain_event'
|
||||
},
|
||||
source_host_uuid=uuidsentinel.fake_host,
|
||||
generated_time=NOW, status="new",
|
||||
notification_uuid=uuidsentinel.fake_notification)
|
||||
|
||||
self.engine.process_notification(self.context,
|
||||
notification=notification)
|
||||
self.assertEqual("ignored", notification.status)
|
||||
|
||||
@mock.patch("masakari.engine.drivers.taskflow."
|
||||
"TaskFlowDriver.execute_instance_failure")
|
||||
@mock.patch.object(notification_obj.Notification, "save")
|
||||
|
Loading…
Reference in New Issue
Block a user