From fa2d817839886f52d6f78fac092e3677f06c18eb Mon Sep 17 00:00:00 2001 From: Duc Truong Date: Mon, 26 Oct 2020 18:02:57 +0000 Subject: [PATCH] Shutdown VM before destructive update action - Shutdown VM before a profile update is performed if the VM is in active state. - Start VM after profile update is completed if the VM is in shutdown state. - Update health policy to be disabled before cluster update, cluster recover and cluster node replace and enabled again after the operation has completed. - Add cluster.stop_timeout_before_update cluster config option to specify custom timeout for shutting down a VM as part of profile update. Change-Id: I9744a1c7990e7a01efe5f9e200eddc920916b4ac --- senlin/engine/actions/cluster_action.py | 15 +- senlin/policies/health_policy.py | 23 +- senlin/profiles/os/nova/server.py | 89 ++++- senlin/tests/drivers/os_test/nova_v2.py | 5 +- .../tests/unit/engine/actions/test_update.py | 102 ++++- .../tests/unit/policies/test_health_policy.py | 59 ++- .../unit/profiles/test_nova_server_update.py | 371 ++++++++++++++++-- 7 files changed, 614 insertions(+), 50 deletions(-) diff --git a/senlin/engine/actions/cluster_action.py b/senlin/engine/actions/cluster_action.py index 5ab94857c..2dbb05322 100755 --- a/senlin/engine/actions/cluster_action.py +++ b/senlin/engine/actions/cluster_action.py @@ -240,16 +240,17 @@ class ClusterAction(base.Action): for node_set in plan: child = [] nodes = list(node_set) + nodes.sort() for node in nodes: kwargs = { 'name': 'node_update_%s' % node[:8], 'cluster_id': self.entity.id, 'cause': consts.CAUSE_DERIVED, - 'inputs': { - 'new_profile_id': profile_id, - }, + 'inputs': self.entity.config, } + kwargs['inputs']['new_profile_id'] = profile_id + action_id = base.Action.create(self.context, node, consts.NODE_UPDATE, **kwargs) child.append(action_id) @@ -299,6 +300,14 @@ class ClusterAction(base.Action): profile_only = self.inputs.get('profile_only') if config is not None: + # make sure config values are valid + try: + stop_timeout = config['cluster.stop_timeout_before_update'] + config['cluster.stop_timeout_before_update'] = int( + stop_timeout) + except Exception as e: + return self.RES_ERROR, str(e) + self.entity.config = config if name is not None: self.entity.name = name diff --git a/senlin/policies/health_policy.py b/senlin/policies/health_policy.py index cfa234144..920615179 100644 --- a/senlin/policies/health_policy.py +++ b/senlin/policies/health_policy.py @@ -46,10 +46,16 @@ class HealthPolicy(base.Policy): ('BEFORE', consts.CLUSTER_DEL_NODES), ('BEFORE', consts.CLUSTER_SCALE_IN), ('BEFORE', consts.CLUSTER_RESIZE), + ('BEFORE', consts.CLUSTER_UPDATE), + ('BEFORE', consts.CLUSTER_RECOVER), + ('BEFORE', consts.CLUSTER_REPLACE_NODES), ('BEFORE', consts.NODE_DELETE), ('AFTER', consts.CLUSTER_DEL_NODES), ('AFTER', consts.CLUSTER_SCALE_IN), ('AFTER', consts.CLUSTER_RESIZE), + ('AFTER', consts.CLUSTER_UPDATE), + ('AFTER', consts.CLUSTER_RECOVER), + ('AFTER', consts.CLUSTER_REPLACE_NODES), ('AFTER', consts.NODE_DELETE), ] @@ -410,9 +416,10 @@ class HealthPolicy(base.Policy): def pre_op(self, cluster_id, action, **args): """Hook before action execution. - One of the task for this routine is to disable health policy if the - action is a request that will shrink the cluster. The reason is that - the policy may attempt to recover nodes that are to be deleted. + Disable health policy for actions that modify cluster nodes (e.g. + scale in, delete nodes, cluster update, cluster recover and cluster + replace nodes). + For all other actions, set the health policy data in the action data. :param cluster_id: The ID of the target cluster. :param action: The action to be examined. @@ -421,7 +428,10 @@ class HealthPolicy(base.Policy): """ if action.action in (consts.CLUSTER_SCALE_IN, consts.CLUSTER_DEL_NODES, - consts.NODE_DELETE): + consts.NODE_DELETE, + consts.CLUSTER_UPDATE, + consts.CLUSTER_RECOVER, + consts.CLUSTER_REPLACE_NODES): health_manager.disable(cluster_id) return True @@ -467,7 +477,10 @@ class HealthPolicy(base.Policy): """ if action.action in (consts.CLUSTER_SCALE_IN, consts.CLUSTER_DEL_NODES, - consts.NODE_DELETE): + consts.NODE_DELETE, + consts.CLUSTER_UPDATE, + consts.CLUSTER_RECOVER, + consts.CLUSTER_REPLACE_NODES): health_manager.enable(cluster_id) return True diff --git a/senlin/profiles/os/nova/server.py b/senlin/profiles/os/nova/server.py index a826c68e8..616322d55 100644 --- a/senlin/profiles/os/nova/server.py +++ b/senlin/profiles/os/nova/server.py @@ -331,6 +331,7 @@ class ServerProfile(base.Profile): def __init__(self, type_name, name, **kwargs): super(ServerProfile, self).__init__(type_name, name, **kwargs) self.server_id = None + self.stop_timeout = cfg.CONF.default_nova_timeout def _validate_az(self, obj, az_name, reason=None): try: @@ -1106,7 +1107,7 @@ class ServerProfile(base.Profile): :param obj: The node object to operate on. :param old_flavor: The identity of the current flavor. :param new_flavor: The identity of the new flavor. - :returns: ``None``. + :returns: Returns true if the flavor was updated or false otherwise. :raises: `EResourceUpdate` when operation was a failure. """ old_flavor = self.properties[self.FLAVOR] @@ -1115,7 +1116,23 @@ class ServerProfile(base.Profile): oldflavor = self._validate_flavor(obj, old_flavor, 'update') newflavor = self._validate_flavor(obj, new_flavor, 'update') if oldflavor.id == newflavor.id: - return + return False + + try: + # server has to be active or stopped in order to resize + # stop server if it is active + server = cc.server_get(obj.physical_id) + if server.status == consts.VS_ACTIVE: + cc.server_stop(obj.physical_id) + cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF, + timeout=self.stop_timeout) + elif server.status != consts.VS_SHUTOFF: + raise exc.InternalError( + message='Server needs to be ACTIVE or STOPPED in order to' + ' update flavor.') + except exc.InternalError as ex: + raise exc.EResourceUpdate(type='server', id=obj.physical_id, + message=str(ex)) try: cc.server_resize(obj.physical_id, newflavor.id) @@ -1123,7 +1140,13 @@ class ServerProfile(base.Profile): except exc.InternalError as ex: msg = str(ex) try: - cc.server_resize_revert(obj.physical_id) + server = cc.server_get(obj.physical_id) + if server.status == 'RESIZE': + cc.server_resize_revert(obj.physical_id) + cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF) + + # start server back up in case of exception during resize + cc.server_start(obj.physical_id) cc.wait_for_server(obj.physical_id, consts.VS_ACTIVE) except exc.InternalError as ex1: msg = str(ex1) @@ -1132,11 +1155,13 @@ class ServerProfile(base.Profile): try: cc.server_resize_confirm(obj.physical_id) - cc.wait_for_server(obj.physical_id, consts.VS_ACTIVE) + cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF) except exc.InternalError as ex: raise exc.EResourceUpdate(type='server', id=obj.physical_id, message=str(ex)) + return True + def _update_image(self, obj, new_profile, new_name, new_password): """Update image used by server node. @@ -1172,9 +1197,20 @@ class ServerProfile(base.Profile): return False try: + # server has to be active or stopped in order to resize + # stop server if it is active + if server.status == consts.VS_ACTIVE: + driver.server_stop(obj.physical_id) + driver.wait_for_server(obj.physical_id, consts.VS_SHUTOFF, + timeout=self.stop_timeout) + elif server.status != consts.VS_SHUTOFF: + raise exc.InternalError( + message='Server needs to be ACTIVE or STOPPED in order to' + ' update image.') + driver.server_rebuild(obj.physical_id, new_image_id, new_name, new_password) - driver.wait_for_server(obj.physical_id, consts.VS_ACTIVE) + driver.wait_for_server(obj.physical_id, consts.VS_SHUTOFF) except exc.InternalError as ex: raise exc.EResourceUpdate(type='server', id=obj.physical_id, message=str(ex)) @@ -1262,6 +1298,18 @@ class ServerProfile(base.Profile): nc = self.network(obj) internal_ports = obj.data.get('internal_ports', []) + if networks: + try: + # stop server if it is active + server = cc.server_get(obj.physical_id) + if server.status == consts.VS_ACTIVE: + cc.server_stop(obj.physical_id) + cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF, + timeout=self.stop_timeout) + except exc.InternalError as ex: + raise exc.EResourceUpdate(type='server', id=obj.physical_id, + message=str(ex)) + for n in networks: candidate_ports = self._find_port_by_net_spec( obj, n, internal_ports) @@ -1290,7 +1338,7 @@ class ServerProfile(base.Profile): :param obj: The node object to operate. :param new_profile: The new profile which may contain new network settings. - :return: ``None`` + :return: Returns a tuple of booleans if network was created or deleted. :raises: ``EResourceUpdate`` if there are driver failures. """ networks_current = self.properties[self.NETWORKS] @@ -1308,7 +1356,8 @@ class ServerProfile(base.Profile): # Attach new interfaces if networks_create: self._update_network_add_port(obj, networks_create) - return + + return networks_create, networks_delete def do_update(self, obj, new_profile=None, **params): """Perform update on the server. @@ -1328,6 +1377,15 @@ class ServerProfile(base.Profile): if not self.validate_for_update(new_profile): return False + self.stop_timeout = params.get('cluster.stop_timeout_before_update', + cfg.CONF.default_nova_timeout) + + if not isinstance(self.stop_timeout, int): + raise exc.EResourceUpdate( + type='server', id=obj.physical_id, + message='cluster.stop_timeout_before_update value must be of ' + 'type int.') + name_changed, new_name = self._check_server_name(obj, new_profile) passwd_changed, new_passwd = self._check_password(obj, new_profile) # Update server image: may have side effect of changing server name @@ -1342,13 +1400,26 @@ class ServerProfile(base.Profile): self._update_password(obj, new_passwd) # Update server flavor: note that flavor is a required property - self._update_flavor(obj, new_profile) - self._update_network(obj, new_profile) + flavor_changed = self._update_flavor(obj, new_profile) + network_created, network_deleted = self._update_network( + obj, new_profile) # TODO(Yanyan Hu): Update block_device properties # Update server metadata self._update_metadata(obj, new_profile) + # start server if it was stopped as part of this update operation + if image_changed or flavor_changed or network_deleted: + cc = self.compute(obj) + try: + server = cc.server_get(obj.physical_id) + if server.status == consts.VS_SHUTOFF: + cc.server_start(obj.physical_id) + cc.wait_for_server(obj.physical_id, consts.VS_ACTIVE) + except exc.InternalError as ex: + raise exc.EResourceUpdate(type='server', id=obj.physical_id, + message=str(ex)) + return True def do_get_details(self, obj): diff --git a/senlin/tests/drivers/os_test/nova_v2.py b/senlin/tests/drivers/os_test/nova_v2.py index e525f55c9..fb9394e82 100644 --- a/senlin/tests/drivers/os_test/nova_v2.py +++ b/senlin/tests/drivers/os_test/nova_v2.py @@ -15,6 +15,7 @@ import time from oslo_utils import uuidutils +from senlin.common import consts from senlin.drivers import base from senlin.drivers import sdk @@ -199,7 +200,9 @@ class NovaClient(base.DriverBase): def server_get(self, server): return sdk.FakeResourceObject(self.fake_server_get) - def wait_for_server(self, server, timeout=None): + def wait_for_server(self, server, status=consts.VS_ACTIVE, + failures=None, + interval=2, timeout=None): # sleep for simulated wait time if it was supplied during server_create if server in self.simulated_waits: time.sleep(self.simulated_waits[server]) diff --git a/senlin/tests/unit/engine/actions/test_update.py b/senlin/tests/unit/engine/actions/test_update.py index f51c914f5..b92761309 100644 --- a/senlin/tests/unit/engine/actions/test_update.py +++ b/senlin/tests/unit/engine/actions/test_update.py @@ -110,19 +110,39 @@ class ClusterUpdateTest(base.SenlinTestCase): cluster = mock.Mock(id='FAKE_ID', nodes=[], ACTIVE='ACTIVE') mock_load.return_value = cluster action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx) + config = {'cluster.stop_timeout_before_update': 25} action.inputs = {'name': 'FAKE_NAME', 'metadata': {'foo': 'bar'}, 'timeout': 3600, 'new_profile_id': 'FAKE_PROFILE', - 'profile_only': True} + 'profile_only': True, + 'config': config} res_code, res_msg = action.do_update() self.assertEqual(action.RES_OK, res_code) self.assertEqual('Cluster update completed.', res_msg) + self.assertEqual(action.entity.config, config) cluster.eval_status.assert_called_once_with( action.context, consts.CLUSTER_UPDATE, profile_id='FAKE_PROFILE', updated_at=mock.ANY) + @mock.patch.object(ca.ClusterAction, '_update_nodes') + def test_do_update_invalid_stop_timeout(self, mock_update, mock_load): + cluster = mock.Mock(id='FAKE_ID', nodes=[], ACTIVE='ACTIVE') + mock_load.return_value = cluster + action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx) + config = {'cluster.stop_timeout_before_update': 'abc'} + action.inputs = {'name': 'FAKE_NAME', + 'metadata': {'foo': 'bar'}, + 'timeout': 3600, + 'new_profile_id': 'FAKE_PROFILE', + 'profile_only': True, + 'config': config} + res_code, res_msg = action.do_update() + + self.assertEqual(action.RES_ERROR, res_code) + mock_update.assert_not_called() + def test_do_update_empty_cluster(self, mock_load): cluster = mock.Mock(id='FAKE_ID', nodes=[], ACTIVE='ACTIVE') mock_load.return_value = cluster @@ -149,7 +169,7 @@ class ClusterUpdateTest(base.SenlinTestCase): node1 = mock.Mock(id='node_id1') node2 = mock.Mock(id='node_id2') cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2], - ACTIVE='ACTIVE') + ACTIVE='ACTIVE', config={}) mock_load.return_value = cluster action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx) @@ -157,12 +177,84 @@ class ClusterUpdateTest(base.SenlinTestCase): action.id = 'CLUSTER_ACTION_ID' mock_wait.return_value = (action.RES_OK, 'All dependents completed') mock_action.side_effect = ['NODE_ACTION1', 'NODE_ACTION2'] + kwargs1 = { + 'name': 'node_update_node_id1', + 'cluster_id': cluster.id, + 'cause': consts.CAUSE_DERIVED, + 'inputs': { + 'new_profile_id': 'FAKE_PROFILE', + }, + } + kwargs2 = { + 'name': 'node_update_node_id2', + 'cluster_id': cluster.id, + 'cause': consts.CAUSE_DERIVED, + 'inputs': { + 'new_profile_id': 'FAKE_PROFILE', + }, + } res_code, reason = action._update_nodes('FAKE_PROFILE', [node1, node2]) self.assertEqual(res_code, action.RES_OK) self.assertEqual(reason, 'Cluster update completed.') - self.assertEqual(2, mock_action.call_count) + mock_action.assert_has_calls([ + mock.call(action.context, node1.id, consts.NODE_UPDATE, **kwargs1), + mock.call(action.context, node2.id, consts.NODE_UPDATE, **kwargs2), + ]) + self.assertEqual(1, mock_dep.call_count) + self.assertEqual(2, mock_update.call_count) + mock_start.assert_called_once_with() + + cluster.eval_status.assert_called_once_with( + action.context, consts.CLUSTER_UPDATE, profile_id='FAKE_PROFILE', + updated_at=mock.ANY) + + @mock.patch.object(ao.Action, 'update') + @mock.patch.object(ab.Action, 'create') + @mock.patch.object(dobj.Dependency, 'create') + @mock.patch.object(dispatcher, 'start_action') + @mock.patch.object(ca.ClusterAction, '_wait_for_dependents') + def test_update_nodes_with_config(self, mock_wait, mock_start, mock_dep, + mock_action, mock_update, mock_load): + node1 = mock.Mock(id='node_id1') + node2 = mock.Mock(id='node_id2') + cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2], + ACTIVE='ACTIVE', config={'blah': 'abc'}) + mock_load.return_value = cluster + + action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx) + action.inputs = {'new_profile_id': 'FAKE_PROFILE'} + action.id = 'CLUSTER_ACTION_ID' + mock_wait.return_value = (action.RES_OK, 'All dependents completed') + mock_action.side_effect = ['NODE_ACTION1', 'NODE_ACTION2'] + kwargs1 = { + 'name': 'node_update_node_id1', + 'cluster_id': cluster.id, + 'cause': consts.CAUSE_DERIVED, + 'inputs': { + 'blah': 'abc', + 'new_profile_id': 'FAKE_PROFILE', + }, + } + kwargs2 = { + 'name': 'node_update_node_id2', + 'cluster_id': cluster.id, + 'cause': consts.CAUSE_DERIVED, + 'inputs': { + 'blah': 'abc', + 'new_profile_id': 'FAKE_PROFILE', + }, + } + + res_code, reason = action._update_nodes('FAKE_PROFILE', + [node1, node2]) + self.assertEqual(res_code, action.RES_OK) + self.assertEqual(reason, 'Cluster update completed.') + mock_action.assert_has_calls([ + mock.call(action.context, node1.id, consts.NODE_UPDATE, **kwargs1), + mock.call(action.context, node2.id, consts.NODE_UPDATE, **kwargs2), + ]) self.assertEqual(1, mock_dep.call_count) self.assertEqual(2, mock_update.call_count) mock_start.assert_called_once_with() @@ -181,7 +273,7 @@ class ClusterUpdateTest(base.SenlinTestCase): node1 = mock.Mock(id='node_id1') node2 = mock.Mock(id='node_id2') cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2], - ACTIVE='ACTIVE') + ACTIVE='ACTIVE', config={}) mock_load.return_value = cluster action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx) @@ -220,7 +312,7 @@ class ClusterUpdateTest(base.SenlinTestCase): node1 = mock.Mock(id='node_id1') node2 = mock.Mock(id='node_id2') cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2], - ACTIVE='ACTIVE') + ACTIVE='ACTIVE', config={}) mock_load.return_value = cluster action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx) diff --git a/senlin/tests/unit/policies/test_health_policy.py b/senlin/tests/unit/policies/test_health_policy.py index e4976aa2f..182cd5e09 100644 --- a/senlin/tests/unit/policies/test_health_policy.py +++ b/senlin/tests/unit/policies/test_health_policy.py @@ -287,7 +287,7 @@ class TestHealthPolicy(base.SenlinTestCase): def test_pre_op_default(self): action = mock.Mock(context='action_context', data={}, - action=consts.CLUSTER_RECOVER) + action=consts.CLUSTER_SCALE_OUT) res = self.hp.pre_op(self.cluster.id, action) @@ -310,6 +310,36 @@ class TestHealthPolicy(base.SenlinTestCase): self.assertTrue(res) mock_disable.assert_called_once_with(self.cluster.id) + @mock.patch.object(health_manager, 'disable') + def test_pre_op_update(self, mock_disable): + action = mock.Mock(context='action_context', data={}, + action=consts.CLUSTER_UPDATE) + + res = self.hp.pre_op(self.cluster.id, action) + + self.assertTrue(res) + mock_disable.assert_called_once_with(self.cluster.id) + + @mock.patch.object(health_manager, 'disable') + def test_pre_op_cluster_recover(self, mock_disable): + action = mock.Mock(context='action_context', data={}, + action=consts.CLUSTER_RECOVER) + + res = self.hp.pre_op(self.cluster.id, action) + + self.assertTrue(res) + mock_disable.assert_called_once_with(self.cluster.id) + + @mock.patch.object(health_manager, 'disable') + def test_pre_op_cluster_replace_nodes(self, mock_disable): + action = mock.Mock(context='action_context', data={}, + action=consts.CLUSTER_REPLACE_NODES) + + res = self.hp.pre_op(self.cluster.id, action) + + self.assertTrue(res) + mock_disable.assert_called_once_with(self.cluster.id) + @mock.patch.object(health_manager, 'disable') def test_pre_op_cluster_del_nodes(self, mock_disable): action = mock.Mock(context='action_context', data={}, @@ -394,6 +424,33 @@ class TestHealthPolicy(base.SenlinTestCase): self.assertTrue(res) mock_enable.assert_called_once_with(self.cluster.id) + @mock.patch.object(health_manager, 'enable') + def test_post_op_update(self, mock_enable): + action = mock.Mock(action=consts.CLUSTER_UPDATE) + + res = self.hp.post_op(self.cluster.id, action) + + self.assertTrue(res) + mock_enable.assert_called_once_with(self.cluster.id) + + @mock.patch.object(health_manager, 'enable') + def test_post_op_cluster_recover(self, mock_enable): + action = mock.Mock(action=consts.CLUSTER_RECOVER) + + res = self.hp.post_op(self.cluster.id, action) + + self.assertTrue(res) + mock_enable.assert_called_once_with(self.cluster.id) + + @mock.patch.object(health_manager, 'enable') + def test_post_op_cluster_replace_nodes(self, mock_enable): + action = mock.Mock(action=consts.CLUSTER_REPLACE_NODES) + + res = self.hp.post_op(self.cluster.id, action) + + self.assertTrue(res) + mock_enable.assert_called_once_with(self.cluster.id) + @mock.patch.object(health_manager, 'enable') def test_post_op_cluster_del_nodes(self, mock_enable): action = mock.Mock(action=consts.CLUSTER_DEL_NODES) diff --git a/senlin/tests/unit/profiles/test_nova_server_update.py b/senlin/tests/unit/profiles/test_nova_server_update.py index 2f37727b1..304021783 100644 --- a/senlin/tests/unit/profiles/test_nova_server_update.py +++ b/senlin/tests/unit/profiles/test_nova_server_update.py @@ -13,7 +13,7 @@ import copy from unittest import mock - +from senlin.common import consts from senlin.common import exception as exc from senlin.objects import node as node_obj from senlin.profiles.os.nova import server @@ -275,6 +275,35 @@ class TestNovaServerUpdate(base.SenlinTestCase): def test_update_flavor(self): obj = mock.Mock(physical_id='NOVA_ID') cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE) + profile = server.ServerProfile('t', self.spec) + profile.stop_timeout = 123 + profile._computeclient = cc + x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')] + mock_validate = self.patchobject(profile, '_validate_flavor', + side_effect=x_flavors) + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['flavor'] = 'new_flavor' + new_profile = server.ServerProfile('t1', new_spec) + profile._update_flavor(obj, new_profile) + + mock_validate.assert_has_calls([ + mock.call(obj, 'FLAV', 'update'), + mock.call(obj, 'new_flavor', 'update') + ]) + cc.server_resize.assert_called_once_with('NOVA_ID', '456') + cc.server_resize_confirm.assert_called_once_with('NOVA_ID') + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, + timeout=profile.stop_timeout), + mock.call('NOVA_ID', 'VERIFY_RESIZE'), + mock.call('NOVA_ID', consts.VS_SHUTOFF)]) + + # update flavor on server that is already stopped + def test_update_flavor_stopped_server(self): + obj = mock.Mock(physical_id='NOVA_ID') + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_SHUTOFF) profile = server.ServerProfile('t', self.spec) profile._computeclient = cc x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')] @@ -291,9 +320,9 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_resize.assert_called_once_with('NOVA_ID', '456') cc.server_resize_confirm.assert_called_once_with('NOVA_ID') - cc.wait_for_server.has_calls([ + cc.wait_for_server.assert_has_calls([ mock.call('NOVA_ID', 'VERIFY_RESIZE'), - mock.call('NOVA_ID', 'ACTIVE')]) + mock.call('NOVA_ID', consts.VS_SHUTOFF)]) def test_update_flavor_failed_validation(self): obj = mock.Mock(physical_id='NOVA_ID') @@ -351,16 +380,76 @@ class TestNovaServerUpdate(base.SenlinTestCase): res = profile._update_flavor(obj, new_profile) - self.assertIsNone(res) + self.assertFalse(res) mock_validate.assert_has_calls([ mock.call(obj, 'FLAV', 'update'), mock.call(obj, 'FLAV', 'update'), ]) self.assertEqual(0, cc.server_resize.call_count) + def test_update_flavor_server_stop_failed(self): + obj = mock.Mock(physical_id='NOVA_ID') + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE) + cc.server_stop.side_effect = [ + exc.InternalError(code=500, message='Stop failed')] + profile = server.ServerProfile('t', self.spec) + profile._computeclient = cc + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['flavor'] = 'new_flavor' + new_profile = server.ServerProfile('t1', new_spec) + x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')] + mock_validate = self.patchobject(profile, '_validate_flavor', + side_effect=x_flavors) + + ex = self.assertRaises(exc.EResourceUpdate, + profile._update_flavor, + obj, new_profile) + + mock_validate.assert_has_calls([ + mock.call(obj, 'FLAV', 'update'), + mock.call(obj, 'new_flavor', 'update'), + ]) + cc.server_resize.assert_not_called() + cc.server_resize_revert.assert_not_called() + cc.wait_for_server.assert_not_called() + self.assertEqual("Failed in updating server 'NOVA_ID': Stop " + "failed.", str(ex)) + + def test_update_flavor_server_paused(self): + obj = mock.Mock(physical_id='NOVA_ID') + cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_PAUSED) + profile = server.ServerProfile('t', self.spec) + profile._computeclient = cc + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['flavor'] = 'new_flavor' + new_profile = server.ServerProfile('t1', new_spec) + x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')] + mock_validate = self.patchobject(profile, '_validate_flavor', + side_effect=x_flavors) + + ex = self.assertRaises(exc.EResourceUpdate, + profile._update_flavor, + obj, new_profile) + + mock_validate.assert_has_calls([ + mock.call(obj, 'FLAV', 'update'), + mock.call(obj, 'new_flavor', 'update'), + ]) + cc.server_resize.assert_not_called() + cc.server_resize_revert.assert_not_called() + cc.wait_for_server.assert_not_called() + self.assertEqual("Failed in updating server 'NOVA_ID': Server needs " + "to be ACTIVE or STOPPED in order to update flavor.", + str(ex)) + def test_update_flavor_resize_failed(self): obj = mock.Mock(physical_id='NOVA_ID') cc = mock.Mock() + cc.server_get.side_effect = [ + mock.Mock(status=consts.VS_ACTIVE), + mock.Mock(status='RESIZE')] cc.server_resize.side_effect = [ exc.InternalError(code=500, message='Resize failed')] profile = server.ServerProfile('t', self.spec) @@ -382,15 +471,56 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_resize.assert_called_once_with('NOVA_ID', '456') cc.server_resize_revert.assert_called_once_with('NOVA_ID') - cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE') + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), + mock.call('NOVA_ID', consts.VS_SHUTOFF), + mock.call('NOVA_ID', consts.VS_ACTIVE) + ]) self.assertEqual("Failed in updating server 'NOVA_ID': Resize " "failed.", str(ex)) def test_update_flavor_first_wait_for_server_failed(self): obj = mock.Mock(physical_id='NOVA_ID') cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE) cc.wait_for_server.side_effect = [ + exc.InternalError(code=500, message='TIMEOUT') + ] + + profile = server.ServerProfile('t', self.spec) + profile._computeclient = cc + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['flavor'] = 'new_flavor' + new_profile = server.ServerProfile('t1', new_spec) + x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')] + mock_validate = self.patchobject(profile, '_validate_flavor', + side_effect=x_flavors) + # do it + ex = self.assertRaises(exc.EResourceUpdate, + profile._update_flavor, + obj, new_profile) + + # assertions + mock_validate.assert_has_calls([ + mock.call(obj, 'FLAV', 'update'), + mock.call(obj, 'new_flavor', 'update'), + ]) + cc.server_resize.assert_not_called() + cc.wait_for_server.has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600)]) + self.assertEqual("Failed in updating server 'NOVA_ID': " + "TIMEOUT.", str(ex)) + + def test_update_flavor_second_wait_for_server_failed(self): + obj = mock.Mock(physical_id='NOVA_ID') + cc = mock.Mock() + cc.server_get.side_effect = [ + mock.Mock(status=consts.VS_ACTIVE), + mock.Mock(status='RESIZE')] + cc.wait_for_server.side_effect = [ + None, exc.InternalError(code=500, message='TIMEOUT'), + None, None ] @@ -414,8 +544,11 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_resize.assert_called_once_with('NOVA_ID', '456') cc.wait_for_server.has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), mock.call('NOVA_ID', 'VERIFY_RESIZE'), - mock.call('NOVA_ID', 'ACTIVE')]) + mock.call('NOVA_ID', consts.VS_SHUTOFF), + mock.call('NOVA_ID', consts.VS_ACTIVE), + ]) cc.server_resize_revert.assert_called_once_with('NOVA_ID') self.assertEqual("Failed in updating server 'NOVA_ID': " "TIMEOUT.", str(ex)) @@ -423,6 +556,9 @@ class TestNovaServerUpdate(base.SenlinTestCase): def test_update_flavor_resize_failed_revert_failed(self): obj = mock.Mock(physical_id='NOVA_ID') cc = mock.Mock() + cc.server_get.side_effect = [ + mock.Mock(status=consts.VS_ACTIVE), + mock.Mock(status='RESIZE')] err_resize = exc.InternalError(code=500, message='Resize') cc.server_resize.side_effect = err_resize err_revert = exc.InternalError(code=500, message='Revert') @@ -448,14 +584,16 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_resize.assert_called_once_with('NOVA_ID', '456') cc.server_resize_revert.assert_called_once_with('NOVA_ID') - # the wait_for_server wasn't called - self.assertEqual(0, cc.wait_for_server.call_count) + cc.wait_for_server.has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), + ]) self.assertEqual("Failed in updating server 'NOVA_ID': " "Revert.", str(ex)) def test_update_flavor_confirm_failed(self): obj = mock.Mock(physical_id='NOVA_ID') cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE) err_confirm = exc.InternalError(code=500, message='Confirm') cc.server_resize_confirm.side_effect = err_confirm profile = server.ServerProfile('t', self.spec) @@ -479,13 +617,17 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_resize.assert_called_once_with('NOVA_ID', '456') cc.server_resize_confirm.assert_called_once_with('NOVA_ID') - cc.wait_for_server.assert_called_once_with('NOVA_ID', 'VERIFY_RESIZE') + cc.wait_for_server.has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), + mock.call('NOVA_ID', 'VERIFY_RESIZE'), + ]) self.assertEqual("Failed in updating server 'NOVA_ID': Confirm.", str(ex)) def test_update_flavor_wait_confirm_failed(self): obj = mock.Mock(physical_id='NOVA_ID') cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_SHUTOFF) err_wait = exc.InternalError(code=500, message='Wait') cc.wait_for_server.side_effect = [None, err_wait] profile = server.ServerProfile('t', self.spec) @@ -511,15 +653,16 @@ class TestNovaServerUpdate(base.SenlinTestCase): cc.server_resize_confirm.assert_called_once_with('NOVA_ID') cc.wait_for_server.assert_has_calls([ mock.call('NOVA_ID', 'VERIFY_RESIZE'), - mock.call('NOVA_ID', 'ACTIVE') + mock.call('NOVA_ID', consts.VS_SHUTOFF) ]) self.assertEqual("Failed in updating server 'NOVA_ID': Wait.", str(ex)) def test_update_image(self): profile = server.ServerProfile('t', self.spec) + profile.stop_timeout = 123 x_image = {'id': '123'} - x_server = mock.Mock(image=x_image) + x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE) cc = mock.Mock() cc.server_get.return_value = x_server profile._computeclient = cc @@ -539,7 +682,68 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_rebuild.assert_called_once_with( 'NOVA_ID', '456', 'new_name', 'new_pass') - cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE') + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, + timeout=profile.stop_timeout), + mock.call('NOVA_ID', consts.VS_SHUTOFF), + ]) + + def test_update_image_server_stopped(self): + profile = server.ServerProfile('t', self.spec) + x_image = {'id': '123'} + x_server = mock.Mock(image=x_image, status=consts.VS_SHUTOFF) + cc = mock.Mock() + cc.server_get.return_value = x_server + profile._computeclient = cc + x_new_image = mock.Mock(id='456') + x_images = [x_new_image] + mock_check = self.patchobject(profile, '_validate_image', + side_effect=x_images) + obj = mock.Mock(physical_id='NOVA_ID') + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['image'] = 'new_image' + new_profile = server.ServerProfile('t1', new_spec) + + profile._update_image(obj, new_profile, 'new_name', 'new_pass') + + mock_check.assert_has_calls([ + mock.call(obj, 'new_image', reason='update'), + ]) + cc.server_rebuild.assert_called_once_with( + 'NOVA_ID', '456', 'new_name', 'new_pass') + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF), + ]) + + def test_update_image_server_paused(self): + profile = server.ServerProfile('t', self.spec) + x_image = {'id': '123'} + x_server = mock.Mock(image=x_image, status=consts.VS_PAUSED) + cc = mock.Mock() + cc.server_get.return_value = x_server + profile._computeclient = cc + x_new_image = mock.Mock(id='456') + x_images = [x_new_image] + mock_check = self.patchobject(profile, '_validate_image', + side_effect=x_images) + obj = mock.Mock(physical_id='NOVA_ID') + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['image'] = 'new_image' + new_profile = server.ServerProfile('t1', new_spec) + + ex = self.assertRaises(exc.EResourceUpdate, + profile._update_image, + obj, new_profile, 'new_name', '') + + msg = ("Failed in updating server 'NOVA_ID': Server needs to be ACTIVE" + " or STOPPED in order to update image.") + self.assertEqual(msg, str(ex)) + + mock_check.assert_has_calls([ + mock.call(obj, 'new_image', reason='update'), + ]) + cc.server_rebuild.assert_not_called() + cc.wait_for_server.assert_not_called() def test_update_image_new_image_is_none(self): profile = server.ServerProfile('t', self.spec) @@ -613,7 +817,7 @@ class TestNovaServerUpdate(base.SenlinTestCase): profile = server.ServerProfile('t', old_spec) cc = mock.Mock() profile._computeclient = cc - x_server = mock.Mock(image={'id': '123'}) + x_server = mock.Mock(image={'id': '123'}, status=consts.VS_ACTIVE) cc.server_get.return_value = x_server # this is the new one x_image = mock.Mock(id='456') @@ -631,7 +835,11 @@ class TestNovaServerUpdate(base.SenlinTestCase): cc.server_get.assert_called_once_with('NOVA_ID') cc.server_rebuild.assert_called_once_with( 'NOVA_ID', '456', 'new_name', 'new_pass') - cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE') + cc.wait_for_server.assert_has_calls([ + # first wait is from active to shutoff and has custom timeout + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), + mock.call('NOVA_ID', consts.VS_SHUTOFF), + ]) def test_update_image_old_image_is_none_but_failed(self): old_spec = copy.deepcopy(self.spec) @@ -683,12 +891,42 @@ class TestNovaServerUpdate(base.SenlinTestCase): self.assertEqual(0, cc.server_rebuild.call_count) self.assertEqual(0, cc.wait_for_server.call_count) - def test_update_image_failed_rebuilding(self): + def test_update_image_failed_stopping(self): profile = server.ServerProfile('t', self.spec) x_image = {'id': '123'} x_server = mock.Mock(image=x_image) cc = mock.Mock() cc.server_get.return_value = x_server + cc.server_stop.side_effect = exc.InternalError(message='FAILED') + profile._computeclient = cc + x_new_image = mock.Mock(id='456') + x_images = [x_new_image] + mock_check = self.patchobject(profile, '_validate_image', + side_effect=x_images) + obj = mock.Mock(physical_id='NOVA_ID') + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['image'] = 'new_image' + new_profile = server.ServerProfile('t1', new_spec) + + ex = self.assertRaises(exc.EResourceUpdate, + profile._update_image, + obj, new_profile, 'new_name', 'new_pass') + + self.assertEqual("Failed in updating server 'NOVA_ID': Server needs to" + " be ACTIVE or STOPPED in order to update image.", + str(ex)) + mock_check.assert_has_calls([ + mock.call(obj, 'new_image', reason='update'), + ]) + cc.server_rebuild.assert_not_called() + cc.wait_for_server.assert_not_called() + + def test_update_image_failed_rebuilding(self): + profile = server.ServerProfile('t', self.spec) + x_image = {'id': '123'} + x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE) + cc = mock.Mock() + cc.server_get.return_value = x_server cc.server_rebuild.side_effect = exc.InternalError(message='FAILED') profile._computeclient = cc x_new_image = mock.Mock(id='456') @@ -711,12 +949,14 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_rebuild.assert_called_once_with( 'NOVA_ID', '456', 'new_name', 'new_pass') - self.assertEqual(0, cc.wait_for_server.call_count) + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), + ]) - def test_update_image_failed_waiting(self): + def test_update_image_failed_first_waiting(self): profile = server.ServerProfile('t', self.spec) x_image = {'id': '123'} - x_server = mock.Mock(image=x_image) + x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE) cc = mock.Mock() cc.server_get.return_value = x_server cc.wait_for_server.side_effect = exc.InternalError(message='TIMEOUT') @@ -730,6 +970,38 @@ class TestNovaServerUpdate(base.SenlinTestCase): new_spec['properties']['image'] = 'new_image' new_profile = server.ServerProfile('t1', new_spec) + ex = self.assertRaises(exc.EResourceUpdate, + profile._update_image, + obj, new_profile, 'new_name', 'new_pass') + + self.assertEqual("Failed in updating server 'NOVA_ID': TIMEOUT.", + str(ex)) + mock_check.assert_has_calls([ + mock.call(obj, 'new_image', reason='update'), + ]) + cc.server_rebuild.assert_not_called() + cc.wait_for_server.assert_called_once_with( + 'NOVA_ID', consts.VS_SHUTOFF, timeout=600) + + def test_update_image_failed_second_waiting(self): + profile = server.ServerProfile('t', self.spec) + x_image = {'id': '123'} + x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE) + cc = mock.Mock() + cc.server_get.return_value = x_server + cc.wait_for_server.side_effect = [ + None, + exc.InternalError(message='TIMEOUT')] + profile._computeclient = cc + x_new_image = mock.Mock(id='456') + x_images = [x_new_image] + mock_check = self.patchobject(profile, '_validate_image', + side_effect=x_images) + obj = mock.Mock(physical_id='NOVA_ID') + new_spec = copy.deepcopy(self.spec) + new_spec['properties']['image'] = 'new_image' + new_profile = server.ServerProfile('t1', new_spec) + ex = self.assertRaises(exc.EResourceUpdate, profile._update_image, obj, new_profile, 'new_name', 'new_pass') @@ -741,7 +1013,9 @@ class TestNovaServerUpdate(base.SenlinTestCase): ]) cc.server_rebuild.assert_called_once_with( 'NOVA_ID', '456', 'new_name', 'new_pass') - cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE') + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600), + mock.call('NOVA_ID', consts.VS_SHUTOFF)]) def test_create_interfaces(self): cc = mock.Mock() @@ -847,11 +1121,13 @@ class TestNovaServerUpdate(base.SenlinTestCase): def test_delete_interfaces(self): cc = mock.Mock() + cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE) nc = mock.Mock() net1 = mock.Mock(id='net1') nc.network_get.return_value = net1 nc.port_find.return_value = mock.Mock(id='port3', status='DOWN') profile = server.ServerProfile('t', self.spec) + profile.stop_timeout = 232 profile._computeclient = cc profile._networkclient = nc obj = mock.Mock(physical_id='NOVA_ID', data={'internal_ports': [ @@ -874,6 +1150,10 @@ class TestNovaServerUpdate(base.SenlinTestCase): nc.network_get.assert_has_calls([ mock.call('net1'), mock.call('net1') ]) + cc.wait_for_server.assert_has_calls([ + mock.call('NOVA_ID', consts.VS_SHUTOFF, + timeout=profile.stop_timeout), + ]) cc.server_interface_delete.assert_has_calls([ mock.call('port1', 'NOVA_ID'), mock.call('port2', 'NOVA_ID'), @@ -935,9 +1215,11 @@ class TestNovaServerUpdate(base.SenlinTestCase): ] new_profile = server.ServerProfile('t1', new_spec) - res = profile._update_network(obj, new_profile) + networks_created, networks_deleted = profile._update_network( + obj, new_profile) - self.assertIsNone(res) + self.assertTrue(networks_created) + self.assertTrue(networks_deleted) networks_create = [ {'floating_network': None, 'network': 'net1', 'fixed_ip': 'ip2', @@ -974,10 +1256,14 @@ class TestNovaServerUpdate(base.SenlinTestCase): mock_check_name.return_value = True, 'NEW_NAME' mock_check_password.return_value = True, 'NEW_PASSWORD' mock_update_image.return_value = False + mock_update_flavor.return_value = False + mock_update_network.return_value = False, False obj = mock.Mock(physical_id='FAKE_ID') profile = server.ServerProfile('t', self.spec) profile._computeclient = mock.Mock() + profile._computeclient.server_get = mock.Mock() + profile._computeclient.server_start = mock.Mock() new_profile = server.ServerProfile('t', self.spec) res = profile.do_update(obj, new_profile) @@ -1007,6 +1293,7 @@ class TestNovaServerUpdate(base.SenlinTestCase): mock_update_password): mock_check_name.return_value = False, 'NEW_NAME' mock_check_password.return_value = False, 'OLD_PASS' + mock_update_network.return_value = False, False obj = mock.Mock(physical_id='NOVA_ID') profile = server.ServerProfile('t', self.spec) @@ -1083,6 +1370,9 @@ class TestNovaServerUpdate(base.SenlinTestCase): profile = server.ServerProfile('t', self.spec) profile._computeclient = mock.Mock() + profile._computeclient.server_get = mock.Mock() + profile._computeclient.server_get.return_value = mock.Mock( + status=consts.VS_SHUTOFF) new_spec = copy.deepcopy(self.spec) new_spec['properties']['image'] = 'FAKE_IMAGE_NEW' new_profile = server.ServerProfile('t', new_spec) @@ -1094,6 +1384,10 @@ class TestNovaServerUpdate(base.SenlinTestCase): obj, new_profile, 'OLD_NAME', 'OLD_PASS') self.assertEqual(0, mock_update_name.call_count) self.assertEqual(0, mock_update_password.call_count) + profile._computeclient.server_get.assert_called_once_with( + obj.physical_id) + profile._computeclient.server_start.assert_called_once_with( + obj.physical_id) @mock.patch.object(server.ServerProfile, '_update_flavor') @mock.patch.object(server.ServerProfile, '_update_name') @@ -1129,10 +1423,11 @@ class TestNovaServerUpdate(base.SenlinTestCase): @mock.patch.object(server.ServerProfile, '_update_flavor') def test_do_update_update_flavor_succeeded(self, mock_update_flavor): + mock_update_flavor.return_value = True obj = mock.Mock(physical_id='FAKE_ID') profile = server.ServerProfile('t', self.spec) x_image = {'id': '123'} - x_server = mock.Mock(image=x_image) + x_server = mock.Mock(image=x_image, status=consts.VS_SHUTOFF) cc = mock.Mock() cc.server_get.return_value = x_server gc = mock.Mock() @@ -1146,6 +1441,7 @@ class TestNovaServerUpdate(base.SenlinTestCase): self.assertTrue(res) mock_update_flavor.assert_called_with(obj, new_profile) gc.image_find.assert_called_with('FAKE_IMAGE', False) + cc.server_start.assert_called_once_with(obj.physical_id) @mock.patch.object(server.ServerProfile, '_update_flavor') def test_do_update_update_flavor_failed(self, mock_update_flavor): @@ -1155,7 +1451,7 @@ class TestNovaServerUpdate(base.SenlinTestCase): obj = mock.Mock(physical_id='NOVA_ID') profile = server.ServerProfile('t', self.spec) x_image = {'id': '123'} - x_server = mock.Mock(image=x_image) + x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE) cc = mock.Mock() cc.server_get.return_value = x_server gc = mock.Mock() @@ -1179,10 +1475,10 @@ class TestNovaServerUpdate(base.SenlinTestCase): @mock.patch.object(server.ServerProfile, '_update_network') def test_do_update_update_network_succeeded( self, mock_update_network, mock_update_flavor): - mock_update_network.return_value = True + mock_update_network.return_value = True, True profile = server.ServerProfile('t', self.spec) x_image = {'id': '123'} - x_server = mock.Mock(image=x_image) + x_server = mock.Mock(image=x_image, status=consts.VS_SHUTOFF) cc = mock.Mock() gc = mock.Mock() cc.server_get.return_value = x_server @@ -1197,10 +1493,16 @@ class TestNovaServerUpdate(base.SenlinTestCase): ] new_profile = server.ServerProfile('t', new_spec) - res = profile.do_update(obj, new_profile) + params = {'cluster.stop_timeout_before_update': 134} + + res = profile.do_update(obj, new_profile=new_profile, **params) + self.assertTrue(res) gc.image_find.assert_called_with('FAKE_IMAGE', False) mock_update_network.assert_called_with(obj, new_profile) + cc.server_start.assert_called_once_with(obj.physical_id) + self.assertEqual(profile.stop_timeout, + params['cluster.stop_timeout_before_update']) @mock.patch.object(server.ServerProfile, '_update_password') @mock.patch.object(server.ServerProfile, '_check_password') @@ -1267,3 +1569,20 @@ class TestNovaServerUpdate(base.SenlinTestCase): res = profile.do_update(node_obj, new_profile) self.assertFalse(res) + + def test_do_update_invalid_stop_timeout(self): + profile = server.ServerProfile('t', self.spec) + profile._computeclient = mock.Mock() + node_obj = mock.Mock(physical_id='NOVA_ID') + new_spec = copy.deepcopy(self.spec) + new_profile = server.ServerProfile('t', new_spec) + + params = {'cluster.stop_timeout_before_update': '123'} + ex = self.assertRaises(exc.EResourceUpdate, + profile.do_update, + node_obj, new_profile, **params) + + self.assertEqual("Failed in updating server 'NOVA_ID': " + "cluster.stop_timeout_before_update value must be of " + "type int.", + str(ex))