Ironic Driver: override get_serial_console()

This implements Nova-compatible serial console for Ironic in
Nova scope.

[Nova] blueprint ironic-serial-console-support
[Ironic] spec
    https://review.openstack.org/319505
[Ironic] related patches
    https://review.openstack.org/#/c/293873/
    https://review.openstack.org/#/c/328168/
[Ironic] Related-Bug: #1553083

Change-Id: I38e803021d71fc0760a8ae99b3e97dd0aecb5088

doc/source/support-matrix.ini

@@ -810,7 +810,7 @@ driver-impl-libvirt-lxc=unknown
 driver-impl-libvirt-xen=unknown
 driver-impl-vmware=missing
 driver-impl-hyperv=complete
-driver-impl-ironic=missing
+driver-impl-ironic=complete
 driver-impl-libvirt-vz-vm=missing
 driver-impl-libvirt-vz-ct=missing
 

nova/conf/ironic.py

@@ -90,6 +90,12 @@ Related options:
 
 * api_max_retries
 """),
+    cfg.IntOpt(
+        'serial_console_state_timeout',
+        default=10,
+        min=0,
+        help='Timeout (seconds) to wait for node serial console state '
+             'changed. Set to 0 to disable timeout.'),
 ]
 
 

nova/conf/serial_console.py

@@ -27,8 +27,8 @@ serial_opt_group = cfg.OptGroup("serial_console",
         title="The serial console feature",
         help="""
 The serial console feature allows you to connect to a guest in case a
-graphical console like VNC, RDP or SPICE is not available. This is only
-supported for the libvirt driver.""")
+graphical console like VNC, RDP or SPICE is not available. This is
+supported for the libvirt and Ironic driver.""")
 
 enabled_opt = cfg.BoolOpt('enabled',
         default=False,

nova/tests/unit/virt/ironic/test_driver.py

@@ -27,6 +27,7 @@ from nova.api.metadata import base as instance_metadata
 from nova.compute import power_state as nova_states
 from nova.compute import task_states
 from nova.compute import vm_states
+from nova.console import type as console_type
 from nova import context as nova_context
 from nova import exception
 from nova import hash_ring
@@ -34,6 +35,7 @@ from nova import objects
 from nova import servicegroup
 from nova import test
 from nova.tests.unit import fake_instance
+from nova.tests.unit import matchers as nova_matchers
 from nova.tests.unit import utils
 from nova.tests.unit.virt.ironic import utils as ironic_utils
 from nova.virt import configdrive
@@ -1918,3 +1920,311 @@ class NodeCacheTestCase(test.NoDBTestCase):
 
         expected_cache = {n.uuid: n for n in nodes[1:]}
         self.assertEqual(expected_cache, self.driver.node_cache)
+
+
+@mock.patch.object(FAKE_CLIENT, 'node')
+class IronicDriverConsoleTestCase(test.NoDBTestCase):
+    @mock.patch.object(cw, 'IronicClientWrapper',
+                       lambda *_: FAKE_CLIENT_WRAPPER)
+    def setUp(self):
+        super(IronicDriverConsoleTestCase, self).setUp()
+
+        self.driver = ironic_driver.IronicDriver(fake.FakeVirtAPI())
+        self.ctx = nova_context.get_admin_context()
+        node_uuid = uuidutils.generate_uuid()
+        self.node = ironic_utils.get_test_node(driver='fake', uuid=node_uuid)
+        self.instance = fake_instance.fake_instance_obj(self.ctx,
+                                                        node=node_uuid)
+
+        # mock retries configs to avoid sleeps and make tests run quicker
+        CONF.set_default('api_max_retries', default=1, group='ironic')
+        CONF.set_default('api_retry_interval', default=0, group='ironic')
+
+        self.stub_out('nova.virt.ironic.driver.IronicDriver.'
+                      '_validate_instance_and_node',
+                      lambda _, inst: self.node)
+
+    def _create_console_data(self, enabled=True, console_type='socat',
+                             url='tcp://127.0.0.1:10000'):
+        return {
+            'console_enabled': enabled,
+            'console_info': {
+                'type': console_type,
+                'url': url
+            }
+        }
+
+    def test__get_node_console_with_reset_success(self, mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'])
+
+        def _fake_set_console_mode(node_uuid, mode):
+            # Set it up so that _fake_get_console() returns 'mode'
+            temp_data['target_mode'] = mode
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+
+        expected = self._create_console_data()['console_info']
+
+        result = self.driver._get_node_console_with_reset(self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertEqual(self.node.uuid, result['node'].uuid)
+        self.assertThat(result['console_info'],
+                        nova_matchers.DictMatches(expected))
+
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test__get_node_console_with_reset_console_disabled(self, mock_log,
+                                                           mock_node):
+        def _fake_log_debug(msg, *args, **kwargs):
+            regex = r'Console is disabled for instance .*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.return_value = \
+            self._create_console_data(enabled=False)
+        mock_log.debug.side_effect = _fake_log_debug
+
+        self.assertRaises(exception.ConsoleNotAvailable,
+                          self.driver._get_node_console_with_reset,
+                          self.instance)
+
+        mock_node.get_console.assert_called_once_with(self.node.uuid)
+        mock_node.set_console_mode.assert_not_called()
+        self.assertTrue(mock_log.debug.called)
+
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test__get_node_console_with_reset_set_mode_failed(self, mock_log,
+                                                          mock_node):
+        def _fake_log_error(msg, *args, **kwargs):
+            regex = r'Failed to set console mode .*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.return_value = self._create_console_data()
+        mock_node.set_console_mode.side_effect = exception.NovaException()
+        mock_log.error.side_effect = _fake_log_error
+
+        self.assertRaises(exception.ConsoleNotAvailable,
+                          self.driver._get_node_console_with_reset,
+                          self.instance)
+
+        mock_node.get_console.assert_called_once_with(self.node.uuid)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertTrue(mock_log.error.called)
+
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test__get_node_console_with_reset_wait_failed(self, mock_log,
+                                                      mock_node):
+        def _fake_get_console(node_uuid):
+            if mock_node.set_console_mode.called:
+                # After the call to set_console_mode(), then _wait_state()
+                # will call _get_console() to check the result.
+                raise exception.NovaException()
+            else:
+                return self._create_console_data()
+
+        def _fake_log_error(msg, *args, **kwargs):
+            regex = r'Failed to acquire console information for instance .*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_log.error.side_effect = _fake_log_error
+
+        self.assertRaises(exception.ConsoleNotAvailable,
+                          self.driver._get_node_console_with_reset,
+                          self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertTrue(mock_log.error.called)
+
+    @mock.patch.object(ironic_driver, '_CONSOLE_STATE_CHECKING_INTERVAL', 0.05)
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test__get_node_console_with_reset_wait_timeout(self, mock_log,
+                                                       mock_node):
+        # Set timeout to a small value to reduce testing time
+        # Note: timeout value is integer, use enforce_type=False to set it
+        # to a floating number.
+        CONF.set_override('serial_console_state_timeout', 0.1,
+                          group='ironic', enforce_type=False)
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'])
+
+        def _fake_set_console_mode(node_uuid, mode):
+            # This causes the _wait_state() will timeout because
+            # the target mode never gets set successfully.
+            temp_data['target_mode'] = not mode
+
+        def _fake_log_error(msg, *args, **kwargs):
+            regex = r'Timeout while waiting for console mode to be set .*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+        mock_log.error.side_effect = _fake_log_error
+
+        self.assertRaises(exception.ConsoleNotAvailable,
+                          self.driver._get_node_console_with_reset,
+                          self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertTrue(mock_log.error.called)
+
+    def test_get_serial_console_socat(self, mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'])
+
+        def _fake_set_console_mode(node_uuid, mode):
+            temp_data['target_mode'] = mode
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+
+        result = self.driver.get_serial_console(self.ctx, self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertIsInstance(result, console_type.ConsoleSerial)
+        self.assertEqual('127.0.0.1', result.host)
+        self.assertEqual(10000, result.port)
+
+    def test_get_serial_console_socat_disabled(self, mock_node):
+        mock_node.get_console.return_value = \
+            self._create_console_data(enabled=False)
+
+        self.assertRaises(exception.ConsoleTypeUnavailable,
+                          self.driver.get_serial_console,
+                          self.ctx, self.instance)
+        mock_node.get_console.assert_called_once_with(self.node.uuid)
+        mock_node.set_console_mode.assert_not_called()
+
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test_get_serial_console_socat_invalid_url(self, mock_log, mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'],
+                                             url='an invalid url')
+
+        def _fake_set_console_mode(node_uuid, mode):
+            temp_data['target_mode'] = mode
+
+        def _fake_log_error(msg, *args, **kwargs):
+            regex = r'Invalid Socat console URL .*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+        mock_log.error.side_effect = _fake_log_error
+
+        self.assertRaises(exception.ConsoleTypeUnavailable,
+                          self.driver.get_serial_console,
+                          self.ctx, self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertTrue(mock_log.error.called)
+
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test_get_serial_console_socat_invalid_url_2(self, mock_log, mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'],
+                                             url='http://abcxyz:1a1b')
+
+        def _fake_set_console_mode(node_uuid, mode):
+            temp_data['target_mode'] = mode
+
+        def _fake_log_error(msg, *args, **kwargs):
+            regex = r'Invalid Socat console URL .*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+        mock_log.error.side_effect = _fake_log_error
+
+        self.assertRaises(exception.ConsoleTypeUnavailable,
+                          self.driver.get_serial_console,
+                          self.ctx, self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertTrue(mock_log.error.called)
+
+    @mock.patch.object(ironic_driver, 'LOG', autospec=True)
+    def test_get_serial_console_socat_unsupported_scheme(self, mock_log,
+                                                         mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'],
+                                             url='ssl://127.0.0.1:10000')
+
+        def _fake_set_console_mode(node_uuid, mode):
+            temp_data['target_mode'] = mode
+
+        def _fake_log_warning(msg, *args, **kwargs):
+            regex = r'Socat serial console only supports \"tcp\".*'
+            self.assertThat(msg, matchers.MatchesRegex(regex))
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+        mock_log.warning.side_effect = _fake_log_warning
+
+        self.assertRaises(exception.ConsoleTypeUnavailable,
+                          self.driver.get_serial_console,
+                          self.ctx, self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertTrue(mock_log.warning.called)
+
+    def test_get_serial_console_socat_tcp6(self, mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'],
+                                             url='tcp://[::1]:10000')
+
+        def _fake_set_console_mode(node_uuid, mode):
+            temp_data['target_mode'] = mode
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+
+        result = self.driver.get_serial_console(self.ctx, self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)
+        self.assertIsInstance(result, console_type.ConsoleSerial)
+        self.assertEqual('::1', result.host)
+        self.assertEqual(10000, result.port)
+
+    def test_get_serial_console_shellinabox(self, mock_node):
+        temp_data = {'target_mode': True}
+
+        def _fake_get_console(node_uuid):
+            return self._create_console_data(enabled=temp_data['target_mode'],
+                                             console_type='shellinabox')
+
+        def _fake_set_console_mode(node_uuid, mode):
+            temp_data['target_mode'] = mode
+
+        mock_node.get_console.side_effect = _fake_get_console
+        mock_node.set_console_mode.side_effect = _fake_set_console_mode
+
+        self.assertRaises(exception.ConsoleTypeUnavailable,
+                          self.driver.get_serial_console,
+                          self.ctx, self.instance)
+
+        self.assertGreater(mock_node.get_console.call_count, 1)
+        self.assertEqual(2, mock_node.set_console_mode.call_count)

nova/virt/ironic/driver.py

@@ -31,6 +31,7 @@ from oslo_service import loopingcall
 from oslo_utils import excutils
 from oslo_utils import importutils
 import six
+import six.moves.urllib.parse as urlparse
 
 from nova.api.metadata import base as instance_metadata
 from nova.compute import arch
@@ -40,6 +41,7 @@ from nova.compute import task_states
 from nova.compute import vm_mode
 from nova.compute import vm_states
 import nova.conf
+from nova.console import type as console_type
 from nova import context as nova_context
 from nova import exception
 from nova import hash_ring
@@ -79,6 +81,9 @@ _NODE_FIELDS = ('uuid', 'power_state', 'target_power_state', 'provision_state',
                 'target_provision_state', 'last_error', 'maintenance',
                 'properties', 'instance_uuid')
 
+# Console state checking interval in seconds
+_CONSOLE_STATE_CHECKING_INTERVAL = 1
+
 
 def map_power_state(state):
     try:
@@ -1225,3 +1230,154 @@ class IronicDriver(virt_driver.ComputeDriver):
         # flat network, go ahead and allow the port to be bound
         return super(IronicDriver, self).network_binding_host_id(
             context, instance)
+
+    def _get_node_console_with_reset(self, instance):
+        """Acquire console information for an instance.
+
+        If the console is enabled, the console will be re-enabled
+        before returning.
+
+        :param instance: nova instance
+        :return: a dictionary with below values
+            { 'node': ironic node
+              'console_info': node console info }
+        :raise ConsoleNotAvailable: if console is unavailable
+            for the instance
+        """
+        node = self._validate_instance_and_node(instance)
+        node_uuid = node.uuid
+
+        def _get_console():
+            """Request ironicclient to acquire node console."""
+            try:
+                return self.ironicclient.call('node.get_console', node_uuid)
+            except (exception.NovaException,  # Retry failed
+                    ironic.exc.InternalServerError,  # Validations
+                    ironic.exc.BadRequest) as e:  # Maintenance
+                LOG.error(_LE('Failed to acquire console information for '
+                              'instance %(inst)s: %(reason)s'),
+                          {'inst': instance.uuid,
+                           'reason': e})
+                raise exception.ConsoleNotAvailable()
+
+        def _wait_state(state):
+            """Wait for the expected console mode to be set on node."""
+            console = _get_console()
+            if console['console_enabled'] == state:
+                raise loopingcall.LoopingCallDone(retvalue=console)
+
+            _log_ironic_polling('set console mode', node, instance)
+
+            # Return False to start backing off
+            return False
+
+        def _enable_console(mode):
+            """Request ironicclient to enable/disable node console."""
+            try:
+                self.ironicclient.call('node.set_console_mode', node_uuid,
+                                       mode)
+            except (exception.NovaException,  # Retry failed
+                    ironic.exc.InternalServerError,  # Validations
+                    ironic.exc.BadRequest) as e:  # Maintenance
+                LOG.error(_LE('Failed to set console mode to "%(mode)s" '
+                              'for instance %(inst)s: %(reason)s'),
+                          {'mode': mode,
+                           'inst': instance.uuid,
+                           'reason': e})
+                raise exception.ConsoleNotAvailable()
+
+            # Waiting for the console state to change (disabled/enabled)
+            try:
+                timer = loopingcall.BackOffLoopingCall(_wait_state, state=mode)
+                return timer.start(
+                    starting_interval=_CONSOLE_STATE_CHECKING_INTERVAL,
+                    timeout=CONF.ironic.serial_console_state_timeout,
+                    jitter=0.5).wait()
+            except loopingcall.LoopingCallTimeOut:
+                LOG.error(_LE('Timeout while waiting for console mode to be '
+                              'set to "%(mode)s" on node %(node)s'),
+                          {'mode': mode,
+                           'node': node_uuid})
+                raise exception.ConsoleNotAvailable()
+
+        # Acquire the console
+        console = _get_console()
+
+        # NOTE: Resetting console is a workaround to force acquiring
+        # console when it has already been acquired by another user/operator.
+        # IPMI serial console does not support multi session, so
+        # resetting console will deactivate any active one without
+        # warning the operator.
+        if console['console_enabled']:
+            try:
+                # Disable console
+                _enable_console(False)
+                # Then re-enable it
+                console = _enable_console(True)
+            except exception.ConsoleNotAvailable:
+                # NOTE: We try to do recover on failure.
+                # But if recover fails, the console may remain in
+                # "disabled" state and cause any new connection
+                # will be refused.
+                console = _enable_console(True)
+
+        if console['console_enabled']:
+            return {'node': node,
+                    'console_info': console['console_info']}
+        else:
+            LOG.debug('Console is disabled for instance %s',
+                      instance.uuid)
+            raise exception.ConsoleNotAvailable()
+
+    def get_serial_console(self, context, instance):
+        """Acquire serial console information.
+
+        :param context: request context
+        :param instance: nova instance
+        :return: ConsoleSerial object
+        :raise ConsoleTypeUnavailable: if serial console is unavailable
+            for the instance
+        """
+        LOG.debug('Getting serial console', instance=instance)
+        try:
+            result = self._get_node_console_with_reset(instance)
+        except exception.ConsoleNotAvailable:
+            raise exception.ConsoleTypeUnavailable(console_type='serial')
+
+        node = result['node']
+        console_info = result['console_info']
+
+        if console_info["type"] != "socat":
+            LOG.warning(_LW('Console type "%(type)s" (of ironic node '
+                            '%(node)s) does not support Nova serial console'),
+                        {'type': console_info["type"],
+                         'node': node.uuid},
+                        instance=instance)
+            raise exception.ConsoleTypeUnavailable(console_type='serial')
+
+        # Parse and check the console url
+        url = urlparse.urlparse(console_info["url"])
+        try:
+            scheme = url.scheme
+            hostname = url.hostname
+            port = url.port
+            if not (scheme and hostname and port):
+                raise AssertionError()
+        except (ValueError, AssertionError):
+            LOG.error(_LE('Invalid Socat console URL "%(url)s" '
+                          '(ironic node %(node)s)'),
+                      {'url': console_info["url"],
+                       'node': node.uuid},
+                      instance=instance)
+            raise exception.ConsoleTypeUnavailable(console_type='serial')
+
+        if scheme == "tcp":
+            return console_type.ConsoleSerial(host=hostname,
+                                              port=port)
+        else:
+            LOG.warning(_LW('Socat serial console only supports "tcp". '
+                            'This URL is "%(url)s" (ironic node %(node)s).'),
+                        {'url': console_info["url"],
+                         'node': node.uuid},
+                        instance=instance)
+            raise exception.ConsoleTypeUnavailable(console_type='serial')

releasenotes/notes/ironic-serial-console-support-82632bd4db6d1fda.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - Adds serial console support to Ironic driver. Nova now supports
+    serial console to Ironic bare metals for Ironic ``socat`` console
+    driver.
+    In order to use this feature, serial console must be configured in Nova
+    and the Ironic ``socat`` console driver must be used and configured
+    in Ironic. Ironic serial console configuration is documented in
+    http://docs.openstack.org/developer/ironic/deploy/console.html.