Ironic Driver: override get_serial_console()

This implements Nova-compatible serial console for Ironic in
Nova scope.

[Nova] blueprint ironic-serial-console-support
[Ironic] spec
    https://review.openstack.org/319505
[Ironic] related patches
    https://review.openstack.org/#/c/293873/
    https://review.openstack.org/#/c/328168/
[Ironic] Related-Bug: #1553083

Change-Id: I38e803021d71fc0760a8ae99b3e97dd0aecb5088
This commit is contained in:
Dao Cong Tien 2016-06-10 15:02:43 +07:00 committed by Matt Riedemann
parent 3d5d66d588
commit c9a64996ec
6 changed files with 484 additions and 3 deletions

View File

@ -810,7 +810,7 @@ driver-impl-libvirt-lxc=unknown
driver-impl-libvirt-xen=unknown
driver-impl-vmware=missing
driver-impl-hyperv=complete
driver-impl-ironic=missing
driver-impl-ironic=complete
driver-impl-libvirt-vz-vm=missing
driver-impl-libvirt-vz-ct=missing

View File

@ -90,6 +90,12 @@ Related options:
* api_max_retries
"""),
cfg.IntOpt(
'serial_console_state_timeout',
default=10,
min=0,
help='Timeout (seconds) to wait for node serial console state '
'changed. Set to 0 to disable timeout.'),
]

View File

@ -27,8 +27,8 @@ serial_opt_group = cfg.OptGroup("serial_console",
title="The serial console feature",
help="""
The serial console feature allows you to connect to a guest in case a
graphical console like VNC, RDP or SPICE is not available. This is only
supported for the libvirt driver.""")
graphical console like VNC, RDP or SPICE is not available. This is
supported for the libvirt and Ironic driver.""")
enabled_opt = cfg.BoolOpt('enabled',
default=False,

View File

@ -27,6 +27,7 @@ from nova.api.metadata import base as instance_metadata
from nova.compute import power_state as nova_states
from nova.compute import task_states
from nova.compute import vm_states
from nova.console import type as console_type
from nova import context as nova_context
from nova import exception
from nova import hash_ring
@ -34,6 +35,7 @@ from nova import objects
from nova import servicegroup
from nova import test
from nova.tests.unit import fake_instance
from nova.tests.unit import matchers as nova_matchers
from nova.tests.unit import utils
from nova.tests.unit.virt.ironic import utils as ironic_utils
from nova.virt import configdrive
@ -1918,3 +1920,311 @@ class NodeCacheTestCase(test.NoDBTestCase):
expected_cache = {n.uuid: n for n in nodes[1:]}
self.assertEqual(expected_cache, self.driver.node_cache)
@mock.patch.object(FAKE_CLIENT, 'node')
class IronicDriverConsoleTestCase(test.NoDBTestCase):
@mock.patch.object(cw, 'IronicClientWrapper',
lambda *_: FAKE_CLIENT_WRAPPER)
def setUp(self):
super(IronicDriverConsoleTestCase, self).setUp()
self.driver = ironic_driver.IronicDriver(fake.FakeVirtAPI())
self.ctx = nova_context.get_admin_context()
node_uuid = uuidutils.generate_uuid()
self.node = ironic_utils.get_test_node(driver='fake', uuid=node_uuid)
self.instance = fake_instance.fake_instance_obj(self.ctx,
node=node_uuid)
# mock retries configs to avoid sleeps and make tests run quicker
CONF.set_default('api_max_retries', default=1, group='ironic')
CONF.set_default('api_retry_interval', default=0, group='ironic')
self.stub_out('nova.virt.ironic.driver.IronicDriver.'
'_validate_instance_and_node',
lambda _, inst: self.node)
def _create_console_data(self, enabled=True, console_type='socat',
url='tcp://127.0.0.1:10000'):
return {
'console_enabled': enabled,
'console_info': {
'type': console_type,
'url': url
}
}
def test__get_node_console_with_reset_success(self, mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'])
def _fake_set_console_mode(node_uuid, mode):
# Set it up so that _fake_get_console() returns 'mode'
temp_data['target_mode'] = mode
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
expected = self._create_console_data()['console_info']
result = self.driver._get_node_console_with_reset(self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertEqual(self.node.uuid, result['node'].uuid)
self.assertThat(result['console_info'],
nova_matchers.DictMatches(expected))
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test__get_node_console_with_reset_console_disabled(self, mock_log,
mock_node):
def _fake_log_debug(msg, *args, **kwargs):
regex = r'Console is disabled for instance .*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.return_value = \
self._create_console_data(enabled=False)
mock_log.debug.side_effect = _fake_log_debug
self.assertRaises(exception.ConsoleNotAvailable,
self.driver._get_node_console_with_reset,
self.instance)
mock_node.get_console.assert_called_once_with(self.node.uuid)
mock_node.set_console_mode.assert_not_called()
self.assertTrue(mock_log.debug.called)
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test__get_node_console_with_reset_set_mode_failed(self, mock_log,
mock_node):
def _fake_log_error(msg, *args, **kwargs):
regex = r'Failed to set console mode .*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.return_value = self._create_console_data()
mock_node.set_console_mode.side_effect = exception.NovaException()
mock_log.error.side_effect = _fake_log_error
self.assertRaises(exception.ConsoleNotAvailable,
self.driver._get_node_console_with_reset,
self.instance)
mock_node.get_console.assert_called_once_with(self.node.uuid)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertTrue(mock_log.error.called)
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test__get_node_console_with_reset_wait_failed(self, mock_log,
mock_node):
def _fake_get_console(node_uuid):
if mock_node.set_console_mode.called:
# After the call to set_console_mode(), then _wait_state()
# will call _get_console() to check the result.
raise exception.NovaException()
else:
return self._create_console_data()
def _fake_log_error(msg, *args, **kwargs):
regex = r'Failed to acquire console information for instance .*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.side_effect = _fake_get_console
mock_log.error.side_effect = _fake_log_error
self.assertRaises(exception.ConsoleNotAvailable,
self.driver._get_node_console_with_reset,
self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertTrue(mock_log.error.called)
@mock.patch.object(ironic_driver, '_CONSOLE_STATE_CHECKING_INTERVAL', 0.05)
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test__get_node_console_with_reset_wait_timeout(self, mock_log,
mock_node):
# Set timeout to a small value to reduce testing time
# Note: timeout value is integer, use enforce_type=False to set it
# to a floating number.
CONF.set_override('serial_console_state_timeout', 0.1,
group='ironic', enforce_type=False)
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'])
def _fake_set_console_mode(node_uuid, mode):
# This causes the _wait_state() will timeout because
# the target mode never gets set successfully.
temp_data['target_mode'] = not mode
def _fake_log_error(msg, *args, **kwargs):
regex = r'Timeout while waiting for console mode to be set .*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
mock_log.error.side_effect = _fake_log_error
self.assertRaises(exception.ConsoleNotAvailable,
self.driver._get_node_console_with_reset,
self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertTrue(mock_log.error.called)
def test_get_serial_console_socat(self, mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'])
def _fake_set_console_mode(node_uuid, mode):
temp_data['target_mode'] = mode
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
result = self.driver.get_serial_console(self.ctx, self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertIsInstance(result, console_type.ConsoleSerial)
self.assertEqual('127.0.0.1', result.host)
self.assertEqual(10000, result.port)
def test_get_serial_console_socat_disabled(self, mock_node):
mock_node.get_console.return_value = \
self._create_console_data(enabled=False)
self.assertRaises(exception.ConsoleTypeUnavailable,
self.driver.get_serial_console,
self.ctx, self.instance)
mock_node.get_console.assert_called_once_with(self.node.uuid)
mock_node.set_console_mode.assert_not_called()
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test_get_serial_console_socat_invalid_url(self, mock_log, mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'],
url='an invalid url')
def _fake_set_console_mode(node_uuid, mode):
temp_data['target_mode'] = mode
def _fake_log_error(msg, *args, **kwargs):
regex = r'Invalid Socat console URL .*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
mock_log.error.side_effect = _fake_log_error
self.assertRaises(exception.ConsoleTypeUnavailable,
self.driver.get_serial_console,
self.ctx, self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertTrue(mock_log.error.called)
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test_get_serial_console_socat_invalid_url_2(self, mock_log, mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'],
url='http://abcxyz:1a1b')
def _fake_set_console_mode(node_uuid, mode):
temp_data['target_mode'] = mode
def _fake_log_error(msg, *args, **kwargs):
regex = r'Invalid Socat console URL .*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
mock_log.error.side_effect = _fake_log_error
self.assertRaises(exception.ConsoleTypeUnavailable,
self.driver.get_serial_console,
self.ctx, self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertTrue(mock_log.error.called)
@mock.patch.object(ironic_driver, 'LOG', autospec=True)
def test_get_serial_console_socat_unsupported_scheme(self, mock_log,
mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'],
url='ssl://127.0.0.1:10000')
def _fake_set_console_mode(node_uuid, mode):
temp_data['target_mode'] = mode
def _fake_log_warning(msg, *args, **kwargs):
regex = r'Socat serial console only supports \"tcp\".*'
self.assertThat(msg, matchers.MatchesRegex(regex))
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
mock_log.warning.side_effect = _fake_log_warning
self.assertRaises(exception.ConsoleTypeUnavailable,
self.driver.get_serial_console,
self.ctx, self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertTrue(mock_log.warning.called)
def test_get_serial_console_socat_tcp6(self, mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'],
url='tcp://[::1]:10000')
def _fake_set_console_mode(node_uuid, mode):
temp_data['target_mode'] = mode
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
result = self.driver.get_serial_console(self.ctx, self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)
self.assertIsInstance(result, console_type.ConsoleSerial)
self.assertEqual('::1', result.host)
self.assertEqual(10000, result.port)
def test_get_serial_console_shellinabox(self, mock_node):
temp_data = {'target_mode': True}
def _fake_get_console(node_uuid):
return self._create_console_data(enabled=temp_data['target_mode'],
console_type='shellinabox')
def _fake_set_console_mode(node_uuid, mode):
temp_data['target_mode'] = mode
mock_node.get_console.side_effect = _fake_get_console
mock_node.set_console_mode.side_effect = _fake_set_console_mode
self.assertRaises(exception.ConsoleTypeUnavailable,
self.driver.get_serial_console,
self.ctx, self.instance)
self.assertGreater(mock_node.get_console.call_count, 1)
self.assertEqual(2, mock_node.set_console_mode.call_count)

View File

@ -31,6 +31,7 @@ from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import importutils
import six
import six.moves.urllib.parse as urlparse
from nova.api.metadata import base as instance_metadata
from nova.compute import arch
@ -40,6 +41,7 @@ from nova.compute import task_states
from nova.compute import vm_mode
from nova.compute import vm_states
import nova.conf
from nova.console import type as console_type
from nova import context as nova_context
from nova import exception
from nova import hash_ring
@ -79,6 +81,9 @@ _NODE_FIELDS = ('uuid', 'power_state', 'target_power_state', 'provision_state',
'target_provision_state', 'last_error', 'maintenance',
'properties', 'instance_uuid')
# Console state checking interval in seconds
_CONSOLE_STATE_CHECKING_INTERVAL = 1
def map_power_state(state):
try:
@ -1225,3 +1230,154 @@ class IronicDriver(virt_driver.ComputeDriver):
# flat network, go ahead and allow the port to be bound
return super(IronicDriver, self).network_binding_host_id(
context, instance)
def _get_node_console_with_reset(self, instance):
"""Acquire console information for an instance.
If the console is enabled, the console will be re-enabled
before returning.
:param instance: nova instance
:return: a dictionary with below values
{ 'node': ironic node
'console_info': node console info }
:raise ConsoleNotAvailable: if console is unavailable
for the instance
"""
node = self._validate_instance_and_node(instance)
node_uuid = node.uuid
def _get_console():
"""Request ironicclient to acquire node console."""
try:
return self.ironicclient.call('node.get_console', node_uuid)
except (exception.NovaException, # Retry failed
ironic.exc.InternalServerError, # Validations
ironic.exc.BadRequest) as e: # Maintenance
LOG.error(_LE('Failed to acquire console information for '
'instance %(inst)s: %(reason)s'),
{'inst': instance.uuid,
'reason': e})
raise exception.ConsoleNotAvailable()
def _wait_state(state):
"""Wait for the expected console mode to be set on node."""
console = _get_console()
if console['console_enabled'] == state:
raise loopingcall.LoopingCallDone(retvalue=console)
_log_ironic_polling('set console mode', node, instance)
# Return False to start backing off
return False
def _enable_console(mode):
"""Request ironicclient to enable/disable node console."""
try:
self.ironicclient.call('node.set_console_mode', node_uuid,
mode)
except (exception.NovaException, # Retry failed
ironic.exc.InternalServerError, # Validations
ironic.exc.BadRequest) as e: # Maintenance
LOG.error(_LE('Failed to set console mode to "%(mode)s" '
'for instance %(inst)s: %(reason)s'),
{'mode': mode,
'inst': instance.uuid,
'reason': e})
raise exception.ConsoleNotAvailable()
# Waiting for the console state to change (disabled/enabled)
try:
timer = loopingcall.BackOffLoopingCall(_wait_state, state=mode)
return timer.start(
starting_interval=_CONSOLE_STATE_CHECKING_INTERVAL,
timeout=CONF.ironic.serial_console_state_timeout,
jitter=0.5).wait()
except loopingcall.LoopingCallTimeOut:
LOG.error(_LE('Timeout while waiting for console mode to be '
'set to "%(mode)s" on node %(node)s'),
{'mode': mode,
'node': node_uuid})
raise exception.ConsoleNotAvailable()
# Acquire the console
console = _get_console()
# NOTE: Resetting console is a workaround to force acquiring
# console when it has already been acquired by another user/operator.
# IPMI serial console does not support multi session, so
# resetting console will deactivate any active one without
# warning the operator.
if console['console_enabled']:
try:
# Disable console
_enable_console(False)
# Then re-enable it
console = _enable_console(True)
except exception.ConsoleNotAvailable:
# NOTE: We try to do recover on failure.
# But if recover fails, the console may remain in
# "disabled" state and cause any new connection
# will be refused.
console = _enable_console(True)
if console['console_enabled']:
return {'node': node,
'console_info': console['console_info']}
else:
LOG.debug('Console is disabled for instance %s',
instance.uuid)
raise exception.ConsoleNotAvailable()
def get_serial_console(self, context, instance):
"""Acquire serial console information.
:param context: request context
:param instance: nova instance
:return: ConsoleSerial object
:raise ConsoleTypeUnavailable: if serial console is unavailable
for the instance
"""
LOG.debug('Getting serial console', instance=instance)
try:
result = self._get_node_console_with_reset(instance)
except exception.ConsoleNotAvailable:
raise exception.ConsoleTypeUnavailable(console_type='serial')
node = result['node']
console_info = result['console_info']
if console_info["type"] != "socat":
LOG.warning(_LW('Console type "%(type)s" (of ironic node '
'%(node)s) does not support Nova serial console'),
{'type': console_info["type"],
'node': node.uuid},
instance=instance)
raise exception.ConsoleTypeUnavailable(console_type='serial')
# Parse and check the console url
url = urlparse.urlparse(console_info["url"])
try:
scheme = url.scheme
hostname = url.hostname
port = url.port
if not (scheme and hostname and port):
raise AssertionError()
except (ValueError, AssertionError):
LOG.error(_LE('Invalid Socat console URL "%(url)s" '
'(ironic node %(node)s)'),
{'url': console_info["url"],
'node': node.uuid},
instance=instance)
raise exception.ConsoleTypeUnavailable(console_type='serial')
if scheme == "tcp":
return console_type.ConsoleSerial(host=hostname,
port=port)
else:
LOG.warning(_LW('Socat serial console only supports "tcp". '
'This URL is "%(url)s" (ironic node %(node)s).'),
{'url': console_info["url"],
'node': node.uuid},
instance=instance)
raise exception.ConsoleTypeUnavailable(console_type='serial')

View File

@ -0,0 +1,9 @@
---
features:
- Adds serial console support to Ironic driver. Nova now supports
serial console to Ironic bare metals for Ironic ``socat`` console
driver.
In order to use this feature, serial console must be configured in Nova
and the Ironic ``socat`` console driver must be used and configured
in Ironic. Ironic serial console configuration is documented in
http://docs.openstack.org/developer/ironic/deploy/console.html.