openstack
/
ironic
Code Issues Proposed changes

Add node console notifications 

This patch adds node console notifications, event types are:
"baremetal.node.console_{set, restore}.{start, end, error}".
Developer documentation updated.

Change-Id: I3b3ac74607fd6e218fdf0ea3ff30964e527db399
Partial-Bug: #1606520
This commit is contained in:
Yuriy Zveryanskyy 2016-11-15 18:17:55 +02:00
parent 876db5a613
commit 294f974fe7
10 changed files with 274 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
doc/source/deploy/notifications.rst
View File

@ -262,6 +262,69 @@ node maintenance notification::
ironic-conductor notifications
------------------------------
Node console notifications
------------------------------
These notifications are emitted by the ironic-conductor service when conductor
service starts or stops console for the node. The notification event types for
a node console are:
* ``baremetal.node.console_set.start``
* ``baremetal.node.console_set.end``
* ``baremetal.node.console_set.error``
* ``baremetal.node.console_restore.start``
* ``baremetal.node.console_restore.end``
* ``baremetal.node.console_restore.error``
``console_set`` action is used when start or stop console is initiated via API
request. The ``console_restore`` action is used when the console was already
enabled, but a driver must restart the console because an ironic-conductor was
restarted. This may also be sent when an ironic-conductor takes over a node
that was being managed by another ironic-conductor. "start" and "end"
notifications have INFO level, "error" has ERROR. Example of node console
notification::
{
"priority": "info",
"payload":{
"ironic_object.namespace":"ironic",
"ironic_object.name":"NodePayload",
"ironic_object.version":"1.0",
"ironic_object.data":{
"clean_step": None,
"console_enabled": True,
"created_at": "2016-01-26T20:41:03+00:00",
"driver": "fake",
"extra": {},
"inspection_finished_at": None,
"inspection_started_at": None,
"instance_info": {},
"instance_uuid": None,
"last_error": None,
"maintenance": False,
"maintenance_reason": None,
"network_interface": "flat",
"name": None,
"power_state": "power off",
"properties": {
"memory_mb": 4096,
"cpu_arch": "x86_64",
"local_gb": 10,
"cpus": 8},
"provision_state": "available",
"provision_updated_at": "2016-01-27T20:41:03+00:00",
"resource_class": None,
"target_power_state": None,
"target_provision_state": None,
"updated_at": "2016-01-27T20:41:03+00:00",
"uuid": "1be26c0b-03f2-4d2e-ae87-c02d7f33c123",
}
},
"event_type":"baremetal.node.console_set.end",
"publisher_id":"ironic-conductor.hostname01"
}
baremetal.node.power_set
------------------------

11
ironic/conductor/base_manager.py
View File

@ -30,10 +30,12 @@ from ironic.common import hash_ring as hash
from ironic.common.i18n import _, _LC, _LE, _LI, _LW
from ironic.common import rpc
from ironic.common import states
from ironic.conductor import notification_utils as notify_utils
from ironic.conductor import task_manager
from ironic.conf import CONF
from ironic.db import api as dbapi
from ironic import objects
from ironic.objects import fields as obj_fields
LOG = log.getLogger(__name__)
@ -383,12 +385,18 @@ class BaseConductorManager(object):
try:
with task_manager.acquire(context, node_uuid, shared=False,
purpose='start console') as task:
notify_utils.emit_console_notification(
task, 'console_restore',
obj_fields.NotificationStatus.START)
try:
LOG.debug('Trying to start console of node %(node)s',
{'node': node_uuid})
task.driver.console.start_console(task)
LOG.info(_LI('Successfully started console of node '
'%(node)s'), {'node': node_uuid})
notify_utils.emit_console_notification(
task, 'console_restore',
obj_fields.NotificationStatus.END)
except Exception as err:
msg = (_('Failed to start console of node %(node)s '
'while starting the conductor, so changing '
@ -401,6 +409,9 @@ class BaseConductorManager(object):
task.node.last_error = msg
task.node.console_enabled = False
task.node.save()
notify_utils.emit_console_notification(
task, 'console_restore',
obj_fields.NotificationStatus.ERROR)
except exception.NodeLocked:
LOG.warning(_LW('Node %(node)s is locked while trying to '
'start console on conductor startup'),

27
ironic/conductor/manager.py
View File

@ -69,6 +69,7 @@ from ironic.conf import CONF
from ironic.drivers import base as drivers_base
from ironic import objects
from ironic.objects import base as objects_base
from ironic.objects import fields
MANAGER_TOPIC = 'ironic.conductor_manager'
@ -1335,7 +1336,10 @@ class ConductorManager(base_manager.BaseConductorManager):
task.driver.deploy.take_over(task)
# NOTE(zhenguo): If console enabled, take over the console session
# as well.
console_error = None
if task.node.console_enabled:
notify_utils.emit_console_notification(
task, 'console_restore', fields.NotificationStatus.START)
try:
task.driver.console.start_console(task)
except Exception as err:
@ -1347,10 +1351,17 @@ class ConductorManager(base_manager.BaseConductorManager):
# back to False and set node's last error.
task.node.last_error = msg
task.node.console_enabled = False
console_error = True
else:
notify_utils.emit_console_notification(
task, 'console_restore', fields.NotificationStatus.END)
# NOTE(lucasagomes): Set the ID of the new conductor managing
# this node
task.node.conductor_affinity = self.conductor.id
task.node.save()
if console_error:
notify_utils.emit_console_notification(
task, 'console_restore', fields.NotificationStatus.ERROR)
@METRICS.timer('ConductorManager._check_cleanwait_timeouts')
@periodics.periodic(spacing=CONF.conductor.check_provision_state_interval)
@ -1510,12 +1521,20 @@ class ConductorManager(base_manager.BaseConductorManager):
'valid_states': states.DELETE_ALLOWED_STATES})
raise exception.InvalidState(msg)
if node.console_enabled:
notify_utils.emit_console_notification(
task, 'console_set', fields.NotificationStatus.START)
try:
task.driver.console.stop_console(task)
except Exception as err:
LOG.error(_LE('Failed to stop console while deleting '
'the node %(node)s: %(err)s.'),
{'node': node.uuid, 'err': err})
notify_utils.emit_console_notification(
task, 'console_set', fields.NotificationStatus.ERROR)
else:
node.console_enabled = False
notify_utils.emit_console_notification(
task, 'console_set', fields.NotificationStatus.END)
node.destroy()
LOG.info(_LI('Successfully deleted node %(node)s.'),
{'node': node.uuid})
@ -1699,6 +1718,8 @@ class ConductorManager(base_manager.BaseConductorManager):
def _set_console_mode(self, task, enabled):
"""Internal method to set console mode on a node."""
node = task.node
notify_utils.emit_console_notification(
task, 'console_set', fields.NotificationStatus.START)
try:
if enabled:
task.driver.console.start_console(task)
@ -1715,11 +1736,15 @@ class ConductorManager(base_manager.BaseConductorManager):
'error': e})
node.last_error = msg
LOG.error(msg)
node.save()
notify_utils.emit_console_notification(
task, 'console_set', fields.NotificationStatus.ERROR)
else:
node.console_enabled = enabled
node.last_error = None
finally:
node.save()
notify_utils.emit_console_notification(
task, 'console_set', fields.NotificationStatus.END)
@METRICS.timer('ConductorManager.update_port')
@messaging.expected_exceptions(exception.NodeLocked,

24
ironic/conductor/notification_utils.py
View File

@ -152,3 +152,27 @@ def emit_provision_set_notification(task, level, status, prev_state,
prev_target=prev_target,
event=event
)
def emit_console_notification(task, action, status):
"""Helper for conductor sending a set console state notification.
:param task: a TaskManager instance.
:param action: Action string to go in the EventType. Must be either
'console_set' or 'console_restore'.
:param status: One of `ironic.objects.fields.NotificationStatus.START`,
END or ERROR.
"""
if status == fields.NotificationStatus.ERROR:
level = fields.NotificationLevel.ERROR
else:
level = fields.NotificationLevel.INFO
_emit_conductor_node_notification(
task,
node_objects.NodeConsoleNotification,
node_objects.NodePayload,
action,
level,
status,
)

11
ironic/objects/node.py
View File

@ -618,3 +618,14 @@ class NodeMaintenanceNotification(notification.NotificationBase):
fields = {
'payload': object_fields.ObjectField('NodePayload')
}
@base.IronicObjectRegistry.register
class NodeConsoleNotification(notification.NotificationBase):
"""Notification emitted when node console state changed."""
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'payload': object_fields.ObjectField('NodePayload')
}

30
ironic/tests/unit/conductor/test_base_manager.py
View File

@ -24,8 +24,10 @@ from ironic.common import driver_factory
from ironic.common import exception
from ironic.conductor import base_manager
from ironic.conductor import manager
from ironic.conductor import notification_utils
from ironic.conductor import task_manager
from ironic import objects
from ironic.objects import fields
from ironic.tests import base as tests_base
from ironic.tests.unit.conductor import mgr_utils
from ironic.tests.unit.db import base as tests_db_base
@ -221,7 +223,8 @@ class ManagerSpawnWorkerTestCase(tests_base.TestCase):
class StartConsolesTestCase(mgr_utils.ServiceSetUpMixin,
tests_db_base.DbTestCase):
def test__start_consoles(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test__start_consoles(self, mock_notify):
obj_utils.create_test_node(self.context,
driver='fake',
console_enabled=True)
@ -241,8 +244,14 @@ class StartConsolesTestCase(mgr_utils.ServiceSetUpMixin,
'start_console') as mock_start_console:
self.service._start_consoles(self.context)
self.assertEqual(2, mock_start_console.call_count)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_restore',
fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_restore',
fields.NotificationStatus.END)])
def test__start_consoles_no_console_enabled(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test__start_consoles_no_console_enabled(self, mock_notify):
obj_utils.create_test_node(self.context,
driver='fake',
console_enabled=False)
@ -251,8 +260,10 @@ class StartConsolesTestCase(mgr_utils.ServiceSetUpMixin,
'start_console') as mock_start_console:
self.service._start_consoles(self.context)
self.assertFalse(mock_start_console.called)
self.assertFalse(mock_notify.called)
def test__start_consoles_failed(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test__start_consoles_failed(self, mock_notify):
test_node = obj_utils.create_test_node(self.context,
driver='fake',
console_enabled=True)
@ -265,9 +276,15 @@ class StartConsolesTestCase(mgr_utils.ServiceSetUpMixin,
test_node.refresh()
self.assertFalse(test_node.console_enabled)
self.assertIsNotNone(test_node.last_error)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_restore',
fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_restore',
fields.NotificationStatus.ERROR)])
@mock.patch.object(notification_utils, 'emit_console_notification')
@mock.patch.object(base_manager, 'LOG')
def test__start_consoles_node_locked(self, log_mock):
def test__start_consoles_node_locked(self, log_mock, mock_notify):
test_node = obj_utils.create_test_node(self.context,
driver='fake',
console_enabled=True,
@ -281,9 +298,11 @@ class StartConsolesTestCase(mgr_utils.ServiceSetUpMixin,
self.assertTrue(test_node.console_enabled)
self.assertIsNone(test_node.last_error)
self.assertTrue(log_mock.warning.called)
self.assertFalse(mock_notify.called)
@mock.patch.object(notification_utils, 'emit_console_notification')
@mock.patch.object(base_manager, 'LOG')
def test__start_consoles_node_not_found(self, log_mock):
def test__start_consoles_node_not_found(self, log_mock, mock_notify):
test_node = obj_utils.create_test_node(self.context,
driver='fake',
console_enabled=True)
@ -298,3 +317,4 @@ class StartConsolesTestCase(mgr_utils.ServiceSetUpMixin,
self.assertTrue(test_node.console_enabled)
self.assertIsNone(test_node.last_error)
self.assertTrue(log_mock.warning.called)
self.assertFalse(mock_notify.called)

89
ironic/tests/unit/conductor/test_manager.py
View File

@ -36,6 +36,7 @@ from ironic.common import images
from ironic.common import states
from ironic.common import swift
from ironic.conductor import manager
from ironic.conductor import notification_utils
from ironic.conductor import task_manager
from ironic.conductor import utils as conductor_utils
from ironic.db import api as dbapi
@ -2652,21 +2653,34 @@ class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, tests_db_base.DbTestCase):
self._stop_service()
spawn_mock.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY)
def test_set_console_mode_enabled(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_set_console_mode_enabled(self, mock_notify):
node = obj_utils.create_test_node(self.context, driver='fake')
self._start_service()
self.service.set_console_mode(self.context, node.uuid, True)
self._stop_service()
node.refresh()
self.assertTrue(node.console_enabled)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.END)])
def test_set_console_mode_disabled(self):
node = obj_utils.create_test_node(self.context, driver='fake')
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_set_console_mode_disabled(self, mock_notify):
node = obj_utils.create_test_node(self.context, driver='fake',
console_enabled=True)
self._start_service()
self.service.set_console_mode(self.context, node.uuid, False)
self._stop_service()
node.refresh()
self.assertFalse(node.console_enabled)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.END)])
def test_set_console_mode_not_supported(self):
node = obj_utils.create_test_node(self.context, driver='fake',
@ -2695,7 +2709,8 @@ class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, tests_db_base.DbTestCase):
# Compare true exception hidden by @messaging.expected_exceptions
self.assertEqual(exception.InvalidParameterValue, exc.exc_info[0])
def test_set_console_mode_start_fail(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_set_console_mode_start_fail(self, mock_notify):
node = obj_utils.create_test_node(self.context, driver='fake',
last_error=None,
console_enabled=False)
@ -2708,8 +2723,14 @@ class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, tests_db_base.DbTestCase):
mock_sc.assert_called_once_with(mock.ANY)
node.refresh()
self.assertIsNotNone(node.last_error)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.ERROR)])
def test_set_console_mode_stop_fail(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_set_console_mode_stop_fail(self, mock_notify):
node = obj_utils.create_test_node(self.context, driver='fake',
last_error=None,
console_enabled=True)
@ -2722,8 +2743,14 @@ class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, tests_db_base.DbTestCase):
mock_sc.assert_called_once_with(mock.ANY)
node.refresh()
self.assertIsNotNone(node.last_error)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.ERROR)])
def test_enable_console_already_enabled(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_enable_console_already_enabled(self, mock_notify):
node = obj_utils.create_test_node(self.context, driver='fake',
console_enabled=True)
self._start_service()
@ -2732,8 +2759,10 @@ class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, tests_db_base.DbTestCase):
self.service.set_console_mode(self.context, node.uuid, True)
self._stop_service()
self.assertFalse(mock_sc.called)
self.assertFalse(mock_notify.called)
def test_disable_console_already_disabled(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_disable_console_already_disabled(self, mock_notify):
node = obj_utils.create_test_node(self.context, driver='fake',
console_enabled=False)
self._start_service()
@ -2742,6 +2771,7 @@ class ConsoleTestCase(mgr_utils.ServiceSetUpMixin, tests_db_base.DbTestCase):
self.service.set_console_mode(self.context, node.uuid, False)
self._stop_service()
self.assertFalse(mock_sc.called)
self.assertFalse(mock_notify.called)
def test_get_console(self):
node = obj_utils.create_test_node(self.context, driver='fake',
@ -2860,7 +2890,8 @@ class DestroyNodeTestCase(mgr_utils.ServiceSetUpMixin,
power_state=states.POWER_OFF)
self.service.destroy_node(self.context, node.uuid)
def test_destroy_node_console_enabled(self):
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_destroy_node_console_enabled(self, mock_notify):
self._start_service()
node = obj_utils.create_test_node(self.context, driver='fake',
console_enabled=True)
@ -2871,6 +2902,30 @@ class DestroyNodeTestCase(mgr_utils.ServiceSetUpMixin,
self.assertRaises(exception.NodeNotFound,
self.dbapi.get_node_by_uuid,
node.uuid)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.END)])
@mock.patch.object(notification_utils, 'emit_console_notification')
def test_destroy_node_console_disable_fail(self, mock_notify):
self._start_service()
node = obj_utils.create_test_node(self.context, driver='fake',
console_enabled=True)
with mock.patch.object(self.driver.console,
'stop_console') as mock_sc:
mock_sc.side_effect = Exception()
self.service.destroy_node(self.context, node.uuid)
mock_sc.assert_called_once_with(mock.ANY)
self.assertRaises(exception.NodeNotFound,
self.dbapi.get_node_by_uuid,
node.uuid)
mock_notify.assert_has_calls(
[mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.START),
mock.call(mock.ANY, 'console_set',
obj_fields.NotificationStatus.ERROR)])
def test_destroy_node_adopt_failed_no_power_change(self):
self._start_service()
@ -5522,12 +5577,14 @@ class DoNodeTakeOverTestCase(mgr_utils.ServiceSetUpMixin,
mock_take_over.assert_called_once_with(mock.ANY)
self.assertFalse(mock_start_console.called)
@mock.patch.object(notification_utils, 'emit_console_notification')
@mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console')
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over')
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare')
def test__do_takeover_with_console_enabled(self, mock_prepare,
mock_take_over,
mock_start_console):
mock_start_console,
mock_notify):
self._start_service()
node = obj_utils.create_test_node(self.context, driver='fake',
console_enabled=True)
@ -5540,13 +5597,20 @@ class DoNodeTakeOverTestCase(mgr_utils.ServiceSetUpMixin,
mock_prepare.assert_called_once_with(mock.ANY)
mock_take_over.assert_called_once_with(mock.ANY)
mock_start_console.assert_called_once_with(mock.ANY)
mock_notify.assert_has_calls(
[mock.call(task, 'console_restore',
obj_fields.NotificationStatus.START),
mock.call(task, 'console_restore',
obj_fields.NotificationStatus.END)])
@mock.patch.object(notification_utils, 'emit_console_notification')
@mock.patch('ironic.drivers.modules.fake.FakeConsole.start_console')
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.take_over')
@mock.patch('ironic.drivers.modules.fake.FakeDeploy.prepare')
def test__do_takeover_with_console_exception(self, mock_prepare,
mock_take_over,
mock_start_console):
mock_start_console,
mock_notify):
self._start_service()
mock_start_console.side_effect = Exception()
node = obj_utils.create_test_node(self.context, driver='fake',
@ -5560,6 +5624,11 @@ class DoNodeTakeOverTestCase(mgr_utils.ServiceSetUpMixin,
mock_prepare.assert_called_once_with(mock.ANY)
mock_take_over.assert_called_once_with(mock.ANY)
mock_start_console.assert_called_once_with(mock.ANY)
mock_notify.assert_has_calls(
[mock.call(task, 'console_restore',
obj_fields.NotificationStatus.START),
mock.call(task, 'console_restore',
obj_fields.NotificationStatus.ERROR)])
@mgr_utils.mock_record_keepalive

26
ironic/tests/unit/conductor/test_notification_utils.py
View File

@ -70,6 +70,32 @@ class TestNotificationUtils(base.DbTestCase):
to_power=states.POWER_ON
)
@mock.patch.object(notif_utils, '_emit_conductor_node_notification')
def test_emit_console_notification(self, mock_cond_emit):
notif_utils.emit_console_notification(
self.task, 'console_set', fields.NotificationStatus.END)
mock_cond_emit.assert_called_once_with(
self.task,
node_objects.NodeConsoleNotification,
node_objects.NodePayload,
'console_set',
fields.NotificationLevel.INFO,
fields.NotificationStatus.END,
)
@mock.patch.object(notif_utils, '_emit_conductor_node_notification')
def test_emit_console_notification_error_status(self, mock_cond_emit):
notif_utils.emit_console_notification(
self.task, 'console_set', fields.NotificationStatus.ERROR)
mock_cond_emit.assert_called_once_with(
self.task,
node_objects.NodeConsoleNotification,
node_objects.NodePayload,
'console_set',
fields.NotificationLevel.ERROR,
fields.NotificationStatus.ERROR,
)
@mock.patch.object(notification, 'mask_secrets')
def test__emit_conductor_node_notification(self, mock_secrets):
mock_notify_method = mock.Mock()

4
ironic/tests/unit/objects/test_objects.py
View File

@ -428,8 +428,10 @@ expected_object_fingerprints = {
'NodeCRUDNotification': '1.0-59acc533c11d306f149846f922739c15',
'NodeCRUDPayload': '1.0-37bb4cdd2c84b59fd6ad0547dbf713a0',
'PortCRUDNotification': '1.0-59acc533c11d306f149846f922739c15',
'PortCRUDPayload': '1.0-88acd98c9b08b4c8810e77793152057b',
'NodeMaintenanceNotification': '1.0-59acc533c11d306f149846f922739c15'
'NodeMaintenanceNotification': '1.0-59acc533c11d306f149846f922739c15',
'NodeConsoleNotification': '1.0-59acc533c11d306f149846f922739c15'
}

6
releasenotes/notes/node-console-notifications-17875f95a378adb4.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- Add notifications for start and stop console on the node.
Event types are
"baremetal.node.console_{set, restore}.{start, end, error}"
For more details, see the developer documentation.