RedfishFirmwareInterface - Unit Tests & More logs

I totally missed Julia's comment in the review, this commit
adds unit tests for the RedfishFirmwareInterface and also more
logs when a specific component is missing.

Change-Id: Ice2c946726103d9957518c2d30ddad3310ee145d
(cherry picked from commit 32c9c74459)

ironic/drivers/modules/redfish/firmware.py

@@ -92,9 +92,13 @@ class RedfishFirmware(base.FirmwareInterface):
         system = redfish_utils.get_system(task.node)
 
         if system.bios_version:
-            bios_fw = {'component': 'bios',
+            bios_fw = {'component': redfish_utils.BIOS,
                        'current_version': system.bios_version}
             settings.append(bios_fw)
+        else:
+            LOG.debug('Could not retrieve BiosVersion in node %(node_uuid)s '
+                      'system %(system)s', {'node_uuid': task.node.uuid,
+                                            'system': system.identity})
 
         # NOTE(iurygregory): normally we only relay on the System to
         # perform actions, but to retrieve the BMC Firmware we need to
@@ -102,9 +106,14 @@ class RedfishFirmware(base.FirmwareInterface):
         try:
             manager = redfish_utils.get_manager(task.node, system)
             if manager.firmware_version:
-                bmc_fw = {'component': 'bmc',
+                bmc_fw = {'component': redfish_utils.BMC,
                           'current_version': manager.firmware_version}
                 settings.append(bmc_fw)
+            else:
+                LOG.debug('Could not retrieve FirmwareVersion in node '
+                          '%(node_uuid)s manager %(manager)s',
+                          {'node_uuid': task.node.uuid,
+                           'manager': manager.identity})
         except exception.RedfishError:
             LOG.warning('No manager available to retrieve Firmware '
                         'from the bmc of node %s', task.node.uuid)
@@ -160,19 +169,17 @@ class RedfishFirmware(base.FirmwareInterface):
         :returns: states.CLEANWAIT if Firmware update with the settings is in
             progress asynchronously of None if it is complete.
         """
+        firmware_utils.validate_firmware_interface_update_args(settings)
         node = task.node
-
         update_service = redfish_utils.get_update_service(node)
 
         LOG.debug('Updating Firmware on node %(node_uuid)s with settings '
                   '%(settings)s',
                   {'node_uuid': node.uuid, 'settings': settings})
-
         self._execute_firmware_update(node, update_service, settings)
 
         fw_upd = settings[0]
         wait_interval = fw_upd.get('wait')
-
         deploy_utils.set_async_step_flags(
             node,
             reboot=True,
@@ -199,8 +206,13 @@ class RedfishFirmware(base.FirmwareInterface):
                   '%(node_uuid)s',
                   {'url': fw_upd['url'], 'component': fw_upd['component'],
                    'node_uuid': node.uuid})
-
-        task_monitor = update_service.simple_update(component_url)
+        try:
+            task_monitor = update_service.simple_update(component_url)
+        except sushy.exceptions.MissingAttributeError as e:
+            LOG.error('The attribute #UpdateService.SimpleUpdate is missing '
+                      'on node %(node)s. Error: %(error)s',
+                      {'node': node.uuid, 'error': e.message})
+            raise exception.RedfishError(error=e)
 
         fw_upd['task_monitor'] = task_monitor.task_monitor_uri
         node.set_driver_internal_info('redfish_fw_updates', settings)
@@ -326,8 +338,7 @@ class RedfishFirmware(base.FirmwareInterface):
             LOG.warning('Unable to communicate with firmware update service '
                         'on node %(node)s. Will try again on the next poll. '
                         'Error: %(error)s',
-                        {'node': node.uuid,
-                         'error': e})
+                        {'node': node.uuid, 'error': e})
             return
 
         wait_start_time = current_update.get('wait_start_time')
@@ -453,3 +464,4 @@ class RedfishFirmware(base.FirmwareInterface):
 
         except exception.IronicException:
             firmware_utils.cleanup(node)
+            raise

ironic/drivers/modules/redfish/firmware_utils.py

@@ -25,6 +25,7 @@ from ironic.common.i18n import _
 from ironic.common import image_service
 from ironic.common import swift
 from ironic.conf import CONF
+from ironic.drivers.modules.redfish import utils as redfish_utils
 
 LOG = log.getLogger(__name__)
 
@@ -65,9 +66,10 @@ _UPDATE_FIRMWARE_SCHEMA = {
 }
 
 _FIRMWARE_INTERFACE_UPDATE_SCHEMA = {
-    "$schema": "http://json-schema.org/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "title": "update_firmware clean step schema",
     "type": "array",
+    "minItems": 1,
     # list of firmware update images
     "items": {
         "type": "object",
@@ -76,7 +78,7 @@ _FIRMWARE_INTERFACE_UPDATE_SCHEMA = {
             "component": {
                 "description": "name of the firmware component to be updated",
                 "type": "string",
-                "minLenght": 1
+                "enum": redfish_utils.FIRMWARE_COMPONENTS
             },
             "url": {
                 "description": "URL for firmware file",

ironic/drivers/modules/redfish/utils.py

@@ -80,6 +80,17 @@ COMMON_PROPERTIES = REQUIRED_PROPERTIES.copy()
 COMMON_PROPERTIES.update(OPTIONAL_PROPERTIES)
 
 
+# All available FIRMWARE COMPONENTS
+BIOS = 'bios'
+"BIOS Firmware Component"
+
+BMC = 'bmc'
+"BMC Firmware Component"
+
+FIRMWARE_COMPONENTS = [BIOS, BMC]
+"""Firmware Components available to update"""
+
+
 def parse_driver_info(node):
     """Parse the information required for Ironic to connect to Redfish.
 

ironic/tests/unit/drivers/modules/redfish/test_firmware.py

@@ -11,10 +11,20 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import datetime
 from unittest import mock
 
 from oslo_utils import importutils
 
+from ironic.common import exception
+from ironic.common import states
+from ironic.conductor import task_manager
+from ironic.conductor import utils as manager_utils
+from ironic.conf import CONF
+from ironic.drivers.modules.redfish import firmware as redfish_fw
+from ironic.drivers.modules.redfish import firmware_utils
+from ironic.drivers.modules.redfish import utils as redfish_utils
+from ironic import objects
 from ironic.tests.unit.db import base as db_base
 from ironic.tests.unit.db import utils as db_utils
 from ironic.tests.unit.objects import utils as obj_utils
@@ -38,3 +48,652 @@ class RedfishFirmwareTestCase(db_base.DbTestCase):
                     enabled_firmware_interfaces=['redfish'])
         self.node = obj_utils.create_test_node(
             self.context, driver='redfish', driver_info=INFO_DICT)
+
+    def test_get_properties(self):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            properties = task.driver.get_properties()
+            for prop in redfish_utils.COMMON_PROPERTIES:
+                self.assertIn(prop, properties)
+
+    @mock.patch.object(redfish_utils, 'parse_driver_info', autospec=True)
+    def test_validate(self, mock_parse_driver_info):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            task.driver.firmware.validate(task)
+            mock_parse_driver_info.assert_called_once_with(task.node)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(objects.FirmwareComponentList,
+                       'sync_firmware_components', autospec=True)
+    def test_missing_all_components(self, sync_fw_cmp_mock, manager_mock,
+                                    system_mock, log_mock):
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            system_mock.return_value.identity = "System1"
+            manager_mock.return_value.identity = "Manager1"
+            system_mock.return_value.bios_version = None
+            manager_mock.return_value.firmware_version = None
+
+            self.assertRaises(exception.UnsupportedDriverExtension,
+                              task.driver.firmware.cache_firmware_components,
+                              task)
+
+            sync_fw_cmp_mock.assert_not_called()
+            error_msg = (
+                'Cannot retrieve firmware for node %s: '
+                'no supported components'
+                % self.node.uuid)
+            log_mock.error.assert_called_once_with(error_msg)
+
+            warning_calls = [
+                mock.call('Could not retrieve BiosVersion in node '
+                          '%(node_uuid)s system %(system)s',
+                          {'node_uuid': self.node.uuid,
+                           'system': "System1"}),
+                mock.call('Could not retrieve FirmwareVersion in node '
+                          '%(node_uuid)s manager %(manager)s',
+                          {'node_uuid': self.node.uuid,
+                           'manager': "Manager1"})]
+            log_mock.debug.assert_has_calls(warning_calls)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(objects.FirmwareComponentList,
+                       'sync_firmware_components', autospec=True)
+    @mock.patch.object(objects, 'FirmwareComponent', spec_set=True,
+                       autospec=True)
+    def test_missing_bios_component(self, fw_cmp_mock, sync_fw_cmp_mock,
+                                    manager_mock, system_mock, log_mock):
+        create_list = [{'component': 'bmc', 'current_version': 'v1.0.0'}]
+        sync_fw_cmp_mock.return_value = (
+            create_list, [], []
+        )
+
+        bmc_component = {'component': 'bmc', 'current_version': 'v1.0.0',
+                         'node_id': self.node.id}
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            system_mock.return_value.identity = "System1"
+            system_mock.return_value.bios_version = None
+            manager_mock.return_value.firmware_version = "v1.0.0"
+
+            task.driver.firmware.cache_firmware_components(task)
+            system_mock.assert_called_once_with(task.node)
+
+            log_mock.debug.assert_called_once_with(
+                'Could not retrieve BiosVersion in node '
+                '%(node_uuid)s system %(system)s',
+                {'node_uuid': self.node.uuid, 'system': 'System1'})
+            sync_fw_cmp_mock.assert_called_once_with(
+                task.context, task.node.id,
+                [{'component': 'bmc', 'current_version': 'v1.0.0'}])
+            self.assertTrue(fw_cmp_mock.called)
+            fw_cmp_mock.assert_called_once_with(task.context, **bmc_component)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(objects.FirmwareComponentList,
+                       'sync_firmware_components', autospec=True)
+    @mock.patch.object(objects, 'FirmwareComponent', spec_set=True,
+                       autospec=True)
+    def test_missing_bmc_component(self, fw_cmp_mock, sync_fw_cmp_mock,
+                                   manager_mock, system_mock, log_mock):
+        create_list = [{'component': 'bios', 'current_version': 'v1.0.0'}]
+        sync_fw_cmp_mock.return_value = (
+            create_list, [], []
+        )
+
+        bios_component = {'component': 'bios', 'current_version': 'v1.0.0',
+                          'node_id': self.node.id}
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            manager_mock.return_value.identity = "Manager1"
+            manager_mock.return_value.firmware_version = None
+            system_mock.return_value.bios_version = "v1.0.0"
+            task.driver.firmware.cache_firmware_components(task)
+
+            log_mock.debug.assert_called_once_with(
+                'Could not retrieve FirmwareVersion in node '
+                '%(node_uuid)s manager %(manager)s',
+                {'node_uuid': self.node.uuid, 'manager': "Manager1"})
+            system_mock.assert_called_once_with(task.node)
+            sync_fw_cmp_mock.assert_called_once_with(
+                task.context, task.node.id,
+                [{'component': 'bios', 'current_version': 'v1.0.0'}])
+            self.assertTrue(fw_cmp_mock.called)
+            fw_cmp_mock.assert_called_once_with(task.context, **bios_component)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_system', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_manager', autospec=True)
+    @mock.patch.object(objects, 'FirmwareComponentList', autospec=True)
+    @mock.patch.object(objects, 'FirmwareComponent', spec_set=True,
+                       autospec=True)
+    def test_create_all_components(self, fw_cmp_mock, fw_cmp_list_mock,
+                                   manager_mock, system_mock, log_mock):
+        create_list = [{'component': 'bios', 'current_version': 'v1.0.0'},
+                       {'component': 'bmc', 'current_version': 'v1.0.0'}]
+        fw_cmp_list_mock.sync_firmware_components.return_value = (
+            create_list, [], []
+        )
+
+        bios_component = {'component': 'bios', 'current_version': 'v1.0.0',
+                          'node_id': self.node.id}
+
+        bmc_component = {'component': 'bmc', 'current_version': 'v1.0.0',
+                         'node_id': self.node.id}
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            manager_mock.return_value.firmware_version = "v1.0.0"
+            system_mock.return_value.bios_version = "v1.0.0"
+            task.driver.firmware.cache_firmware_components(task)
+
+            log_mock.warning.assert_not_called()
+            log_mock.debug.assert_not_called()
+            system_mock.assert_called_once_with(task.node)
+            fw_cmp_list_mock.sync_firmware_components.assert_called_once_with(
+                task.context, task.node.id,
+                [{'component': 'bios', 'current_version': 'v1.0.0'},
+                 {'component': 'bmc', 'current_version': 'v1.0.0'}])
+            fw_cmp_calls = [
+                mock.call(task.context, **bios_component),
+                mock.call().create(),
+                mock.call(task.context, **bmc_component),
+                mock.call().create(),
+            ]
+            fw_cmp_mock.assert_has_calls(fw_cmp_calls)
+
+    @mock.patch.object(redfish_utils, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, '_get_connection', autospec=True)
+    def test_missing_updateservice(self, conn_mock, log_mock):
+        settings = [{'component': 'bmc', 'url': 'http://upfwbmc/v2.0.0'}]
+        conn_mock.side_effect = sushy.exceptions.MissingAttributeError(
+            attribute='UpdateService', resource='redfish/v1')
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            error_msg = ('The attribute UpdateService is missing from the '
+                         'resource redfish/v1')
+            self.assertRaisesRegex(
+                exception.RedfishError, error_msg,
+                task.driver.firmware.update,
+                task, settings)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    def test_missing_simple_update_action(self, update_service_mock, log_mock):
+        settings = [{'component': 'bmc', 'url': 'http://upfwbmc/v2.0.0'}]
+        update_service = update_service_mock.return_value
+        update_service.simple_update.side_effect = \
+            sushy.exceptions.MissingAttributeError(
+                attribute='#UpdateService.SimpleUpdate',
+                resource='redfish/v1/UpdateService')
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+
+            self.assertRaises(
+                exception.RedfishError,
+                task.driver.firmware.update,
+                task, settings)
+            expected_err_msg = (
+                'The attribute #UpdateService.SimpleUpdate is missing '
+                'from the resource redfish/v1/UpdateService')
+            log_mock.error.assert_called_once_with(
+                'The attribute #UpdateService.SimpleUpdate is missing '
+                'on node %(node)s. Error: %(error)s',
+                {'node': self.node.uuid, 'error': expected_err_msg})
+
+            component = settings[0].get('component')
+            url = settings[0].get('url')
+
+            log_call = [
+                mock.call('Updating Firmware on node %(node_uuid)s '
+                          'with settings %(settings)s',
+                          {'node_uuid': self.node.uuid,
+                           'settings': settings}),
+                mock.call('For node %(node)s serving firmware for '
+                          '%(component)s from original location %(url)s',
+                          {'node': self.node.uuid,
+                           'component': component, 'url': url}),
+                mock.call('Applying new firmware %(url)s for '
+                          '%(component)s on node %(node_uuid)s',
+                          {'url': url, 'component': component,
+                           'node_uuid': self.node.uuid})
+            ]
+            log_mock.debug.assert_has_calls(log_call)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    def _test_invalid_settings(self, log_mock):
+        step = self.node.clean_step
+        settings = step['argsinfo'].get('settings', None)
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            self.assertRaises(
+                exception.InvalidParameterValue,
+                task.driver.firmware.update,
+                task, settings)
+            log_mock.debug.assert_not_called()
+
+    def test_invalid_component_in_settings(self):
+        argsinfo = {'settings': [
+            {'component': 'nic', 'url': 'https://nic-update/v1.1.0'}
+        ]}
+        self.node.clean_step = {'priority': 100, 'interface': 'firmware',
+                                'step': 'update',
+                                'argsinfo': argsinfo}
+        self.node.save()
+        self._test_invalid_settings()
+
+    def test_missing_required_field_in_settings(self):
+        argsinfo = {'settings': [
+            {'url': 'https://nic-update/v1.1.0'},
+            {'component': "bmc"}
+        ]}
+        self.node.clean_step = {'priority': 100, 'interface': 'firmware',
+                                'step': 'update',
+                                'argsinfo': argsinfo}
+        self.node.save()
+        self._test_invalid_settings()
+
+    def test_empty_settings(self):
+        argsinfo = {'settings': []}
+        self.node.clean_step = {'priority': 100, 'interface': 'firmware',
+                                'step': 'update',
+                                'argsinfo': argsinfo}
+        self.node.save()
+        self._test_invalid_settings()
+
+    def _generate_new_driver_internal_info(self, components=[], invalid=False,
+                                           add_wait=False, wait=1):
+        bmc_component = {'component': 'bmc', 'url': 'https://bmc/v1.0.1'}
+        bios_component = {'component': 'bios', 'url': 'https://bios/v1.0.1'}
+        if add_wait:
+            wait_start_time = datetime.datetime.utcnow() -\
+                datetime.timedelta(minutes=1)
+            bmc_component['wait_start_time'] = wait_start_time.isoformat()
+            bios_component['wait_start_time'] = wait_start_time.isoformat()
+            bmc_component['wait'] = wait
+            bios_component['wait'] = wait
+
+        self.node.clean_step = {'priority': 100, 'interface': 'bios',
+                                'step': 'apply_configuration',
+                                'argsinfo': {'settings': []}}
+
+        updates = []
+        if 'bmc' in components:
+            self.node.clean_step['argsinfo']['settings'].append(
+                bmc_component)
+            bmc_component['task_monitor'] = '/task/1'
+            updates.append(bmc_component)
+        if 'bios' in components:
+            self.node.clean_step['argsinfo']['settings'].append(
+                bios_component)
+            bios_component['task_monitor'] = '/task/2'
+            updates.append(bios_component)
+
+        if invalid:
+            self.node.provision_state = states.CLEANING
+            self.node.driver_internal_info = {'something': 'else'}
+        else:
+            self.node.provision_state = states.CLEANING
+            self.node.driver_internal_info = {
+                'redfish_fw_updates': updates,
+            }
+        self.node.save()
+
+    @mock.patch.object(task_manager, 'acquire', autospec=True)
+    def _test__query_methods(self, acquire_mock):
+        firmware = redfish_fw.RedfishFirmware()
+        mock_manager = mock.Mock()
+        node_list = [(self.node.uuid, 'redfish', '',
+                      self.node.driver_internal_info)]
+        mock_manager.iter_nodes.return_value = node_list
+        task = mock.Mock(node=self.node,
+                         driver=mock.Mock(firmware=firmware))
+        acquire_mock.return_value = mock.MagicMock(
+            __enter__=mock.MagicMock(return_value=task))
+
+        firmware._check_node_redfish_firmware_update = mock.Mock()
+        firmware._clear_updates = mock.Mock()
+
+        # _query_update_status
+        firmware._query_update_status(mock_manager, self.context)
+        if not self.node.driver_internal_info.get('redfish_fw_updates'):
+            firmware._check_node_redfish_firmware_update.assert_not_called()
+        else:
+            firmware._check_node_redfish_firmware_update.\
+                assert_called_once_with(task)
+
+        # _query_update_failed
+        firmware._query_update_failed(mock_manager, self.context)
+        if not self.node.driver_internal_info.get('redfish_fw_updates'):
+            firmware._clear_updates.assert_not_called()
+        else:
+            firmware._clear_updates.assert_called_once_with(self.node)
+
+    def test_redfish_fw_updates(self):
+        self._generate_new_driver_internal_info(['bmc'])
+        self._test__query_methods()
+
+    def test_redfish_fw_updates_empty(self):
+        self._generate_new_driver_internal_info(invalid=True)
+        self._test__query_methods()
+
+    def _test__check_node_redfish_firmware_update(self):
+        firmware = redfish_fw.RedfishFirmware()
+        firmware._continue_updates = mock.Mock()
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.upgrade_lock = mock.Mock()
+            task.process_event = mock.Mock()
+            firmware._check_node_redfish_firmware_update(task)
+            return task, firmware
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    def test_check_conn_error(self, get_us_mock, log_mock):
+        self._generate_new_driver_internal_info(['bmc'])
+        get_us_mock.side_effect = exception.RedfishConnectionError('Error')
+        try:
+            self._test__check_node_redfish_firmware_update()
+        except exception.RedfishError as e:
+            exception_error = e.kwargs.get('error')
+
+            warning_calls = [
+                mock.call('Unable to communicate with firmware update '
+                          'service on node %(node)s. Will try again on '
+                          'the next poll. Error: %(error)s',
+                          {'node': self.node.uuid,
+                           'error': exception_error})
+            ]
+            log_mock.warning.assert_has_calls(warning_calls)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    def test_check_update_wait_elapsed(self, get_us_mock, log_mock):
+        mock_update_service = mock.Mock()
+        get_us_mock.return_value = mock_update_service
+        self._generate_new_driver_internal_info(['bmc'], add_wait=True)
+
+        task, interface = self._test__check_node_redfish_firmware_update()
+        debug_calls = [
+            mock.call('Finished waiting after firmware update '
+                      '%(firmware_image)s on node %(node)s. '
+                      'Elapsed time: %(seconds)s seconds',
+                      {'firmware_image': 'https://bmc/v1.0.1',
+                       'node': self.node.uuid, 'seconds': 60})]
+        log_mock.debug.assert_has_calls(debug_calls)
+        interface._continue_updates.assert_called_once_with(
+            task,
+            mock_update_service,
+            [{'component': 'bmc', 'url': 'https://bmc/v1.0.1',
+              'task_monitor': '/task/1'}])
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    def test_check_update_still_waiting(self, get_us_mock, log_mock):
+        mock_update_service = mock.Mock()
+        get_us_mock.return_value = mock_update_service
+        self._generate_new_driver_internal_info(
+            ['bios'], add_wait=True, wait=600)
+
+        _, interface = self._test__check_node_redfish_firmware_update()
+        debug_calls = [
+            mock.call('Continuing to wait after firmware update '
+                      '%(firmware_image)s on node %(node)s. '
+                      'Elapsed time: %(seconds)s seconds',
+                      {'firmware_image': 'https://bios/v1.0.1',
+                       'node': self.node.uuid, 'seconds': 60})]
+        log_mock.debug.assert_has_calls(debug_calls)
+        interface._continue_updates.assert_not_called()
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
+    def test_check_update_task_monitor_not_found(self, tm_mock, get_us_mock,
+                                                 log_mock):
+        tm_mock.side_effect = exception.RedfishError()
+        self._generate_new_driver_internal_info(['bios'])
+
+        task, interface = self._test__check_node_redfish_firmware_update()
+        warning_calls = [
+            mock.call('Firmware update completed for node %(node)s, '
+                      'firmware %(firmware_image)s, but success of the '
+                      'update is unknown.  Assuming update was successful.',
+                      {'node': self.node.uuid,
+                       'firmware_image': 'https://bios/v1.0.1'})]
+
+        log_mock.warning.assert_has_calls(warning_calls)
+        interface._continue_updates.assert_called_once_with(
+            task, get_us_mock.return_value,
+            [{'component': 'bios', 'url': 'https://bios/v1.0.1',
+              'task_monitor': '/task/2'}]
+        )
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
+    def test__check_update_in_progress(self, tm_mock, get_us_mock, log_mock):
+        tm_mock.return_value.is_processing = True
+        self._generate_new_driver_internal_info(['bmc'])
+
+        _, interface = self._test__check_node_redfish_firmware_update()
+        debug_calls = [
+            mock.call('Firmware update in progress for node %(node)s, '
+                      'firmware %(firmware_image)s.',
+                      {'node': self.node.uuid,
+                       'firmware_image': 'https://bmc/v1.0.1'})]
+
+        log_mock.debug.assert_has_calls(debug_calls)
+
+        interface._continue_updates.assert_not_called()
+
+    @mock.patch.object(manager_utils, 'cleaning_error_handler', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
+    def test__check_node_firmware_update_fail(self, tm_mock, get_us_mock,
+                                              cleaning_error_handler_mock):
+        mock_sushy_task = mock.Mock()
+        mock_sushy_task.task_state = 'exception'
+        mock_message_unparsed = mock.Mock()
+        mock_message_unparsed.message = None
+        message_mock = mock.Mock()
+        message_mock.message = 'Firmware upgrade failed'
+        messages = mock.MagicMock(return_value=[[mock_message_unparsed],
+                                                [message_mock],
+                                                [message_mock]])
+        mock_sushy_task.messages = messages
+        mock_task_monitor = mock.Mock()
+        mock_task_monitor.is_processing = False
+        mock_task_monitor.get_task.return_value = mock_sushy_task
+        tm_mock.return_value = mock_task_monitor
+        self._generate_new_driver_internal_info(['bmc'])
+
+        task, interface = self._test__check_node_redfish_firmware_update()
+
+        task.upgrade_lock.assert_called_once_with()
+        cleaning_error_handler_mock.assert_called_once()
+        interface._continue_updates.assert_not_called()
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_update_service', autospec=True)
+    @mock.patch.object(redfish_utils, 'get_task_monitor', autospec=True)
+    def test__check_node_firmware_update_done(self, tm_mock, get_us_mock,
+                                              log_mock):
+        task_mock = mock.Mock()
+        task_mock.task_state = sushy.TASK_STATE_COMPLETED
+        task_mock.task_status = sushy.HEALTH_OK
+        message_mock = mock.Mock()
+        message_mock.message = 'Firmware update done'
+        task_mock.messages = [message_mock]
+        mock_task_monitor = mock.Mock()
+        mock_task_monitor.is_processing = False
+        mock_task_monitor.get_task.return_value = task_mock
+        tm_mock.return_value = mock_task_monitor
+        self._generate_new_driver_internal_info(['bmc'])
+
+        task, interface = self._test__check_node_redfish_firmware_update()
+        task.upgrade_lock.assert_called_once_with()
+        info_calls = [
+            mock.call('Firmware update succeeded for node %(node)s, '
+                      'firmware %(firmware_image)s: %(messages)s',
+                      {'node': self.node.uuid,
+                       'firmware_image': 'https://bmc/v1.0.1',
+                       'messages': 'Firmware update done'})]
+
+        log_mock.info.assert_has_calls(info_calls)
+
+        interface._continue_updates.assert_called_once_with(
+            task, get_us_mock.return_value,
+            [{'component': 'bmc', 'url': 'https://bmc/v1.0.1',
+              'task_monitor': '/task/1'}]
+        )
+
+    @mock.patch.object(firmware_utils, 'download_to_temp', autospec=True)
+    @mock.patch.object(firmware_utils, 'stage', autospec=True)
+    def test__stage_firmware_file_https(self, stage_mock, dwl_tmp_mock):
+        CONF.set_override('firmware_source', 'local', 'redfish')
+        firmware_update = {'url': 'https://test1', 'component': 'bmc'}
+        node = mock.Mock()
+        dwl_tmp_mock.return_value = '/tmp/test1'
+        stage_mock.return_value = ('http://staged/test1', 'http')
+
+        firmware = redfish_fw.RedfishFirmware()
+
+        staged_url, needs_cleanup = firmware._stage_firmware_file(
+            node, firmware_update)
+
+        self.assertEqual(staged_url, 'http://staged/test1')
+        self.assertEqual(needs_cleanup, 'http')
+        dwl_tmp_mock.assert_called_with(node, 'https://test1')
+        stage_mock.assert_called_with(node, 'local', '/tmp/test1')
+
+    @mock.patch.object(firmware_utils, 'download_to_temp', autospec=True)
+    @mock.patch.object(firmware_utils, 'stage', autospec=True)
+    @mock.patch.object(firmware_utils, 'get_swift_temp_url', autospec=True)
+    def test__stage_firmware_file_swift(
+            self, get_swift_tmp_url_mock, stage_mock, dwl_tmp_mock):
+        CONF.set_override('firmware_source', 'swift', 'redfish')
+        firmware_update = {'url': 'swift://container/bios.exe',
+                           'component': 'bios'}
+        node = mock.Mock()
+        get_swift_tmp_url_mock.return_value = 'http://temp'
+
+        firmware = redfish_fw.RedfishFirmware()
+
+        staged_url, needs_cleanup = firmware._stage_firmware_file(
+            node, firmware_update)
+
+        self.assertEqual(staged_url, 'http://temp')
+        self.assertIsNone(needs_cleanup)
+        dwl_tmp_mock.assert_not_called()
+        stage_mock.assert_not_called()
+
+    @mock.patch.object(firmware_utils, 'cleanup', autospec=True)
+    @mock.patch.object(firmware_utils, 'download_to_temp', autospec=True)
+    @mock.patch.object(firmware_utils, 'stage', autospec=True)
+    def test__stage_firmware_file_error(self, stage_mock, dwl_tmp_mock,
+                                        cleanup_mock):
+        CONF.set_override('firmware_source', 'local', 'redfish')
+        node = mock.Mock()
+        firmware_update = {'url': 'https://test1', 'component': 'bmc'}
+        dwl_tmp_mock.return_value = '/tmp/test1'
+        stage_mock.side_effect = exception.IronicException
+
+        firmware = redfish_fw.RedfishFirmware()
+        self.assertRaises(exception.IronicException,
+                          firmware._stage_firmware_file, node,
+                          firmware_update)
+        dwl_tmp_mock.assert_called_with(node, 'https://test1')
+        stage_mock.assert_called_with(node, 'local', '/tmp/test1')
+        cleanup_mock.assert_called_with(node)
+
+    def _test_continue_updates(self):
+
+        update_service_mock = mock.Mock()
+        firmware = redfish_fw.RedfishFirmware()
+
+        updates = self.node.driver_internal_info.get('redfish_fw_updates')
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            firmware._continue_updates(
+                task,
+                update_service_mock,
+                updates
+            )
+            return task
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    def test_continue_update_waitting(self, log_mock):
+        self._generate_new_driver_internal_info(['bmc', 'bios'],
+                                                add_wait=True, wait=120)
+        self._test_continue_updates()
+        debug_call = [
+            mock.call('Waiting at %(time)s for %(seconds)s seconds '
+                      'after %(component)s firmware update %(url)s '
+                      'on node %(node)s',
+                      {'time': mock.ANY, 'seconds': 120,
+                       'component': 'bmc', 'url': 'https://bmc/v1.0.1',
+                       'node': self.node.uuid})
+        ]
+        log_mock.debug.assert_has_calls(debug_call)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(manager_utils, 'notify_conductor_resume_clean',
+                       autospec=True)
+    def test_continue_updates_last(self, cond_resume_clean_mock, log_mock):
+        self._generate_new_driver_internal_info(['bmc'])
+        task = self._test_continue_updates()
+
+        cond_resume_clean_mock.assert_called_once_with(task)
+
+        info_call = [
+            mock.call('Firmware updates completed for node %(node)s',
+                      {'node': self.node.uuid})
+        ]
+        log_mock.info.assert_has_calls(info_call)
+
+    @mock.patch.object(redfish_fw, 'LOG', autospec=True)
+    @mock.patch.object(manager_utils, 'node_power_action', autospec=True)
+    def test_continue_updates_more_updates(self, node_power_action_mock,
+                                           log_mock):
+        self._generate_new_driver_internal_info(['bmc', 'bios'])
+
+        task_monitor_mock = mock.Mock()
+        task_monitor_mock.task_monitor_uri = '/task/2'
+        update_service_mock = mock.Mock()
+        update_service_mock.simple_update.return_value = task_monitor_mock
+
+        firmware = redfish_fw.RedfishFirmware()
+        updates = self.node.driver_internal_info.get('redfish_fw_updates')
+
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=True) as task:
+            task.node.save = mock.Mock()
+
+            firmware._continue_updates(task, update_service_mock, updates)
+
+            debug_calls = [
+                mock.call('Applying new firmware %(url)s for '
+                          '%(component)s on node %(node_uuid)s',
+                          {'url': 'https://bios/v1.0.1', 'component': 'bios',
+                           'node_uuid': self.node.uuid})
+            ]
+            log_mock.debug.assert_has_calls(debug_calls)
+            self.assertEqual(
+                [{'component': 'bios', 'url': 'https://bios/v1.0.1',
+                  'task_monitor': '/task/2'}],
+                task.node.driver_internal_info['redfish_fw_updates'])
+            update_service_mock.simple_update.assert_called_once_with(
+                'https://bios/v1.0.1')
+            task.node.save.assert_called_once_with()
+            node_power_action_mock.assert_called_once_with(task, states.REBOOT)