diff --git a/senlin_tempest_plugin/common/compute_client.py b/senlin_tempest_plugin/common/compute_client.py index e5ac46e..927f211 100644 --- a/senlin_tempest_plugin/common/compute_client.py +++ b/senlin_tempest_plugin/common/compute_client.py @@ -13,8 +13,12 @@ # limitations under the License. from oslo_serialization import jsonutils +from oslo_utils import timeutils +from six.moves.urllib import parse as urllib from tempest import config from tempest.lib.common import rest_client +from tempest.lib import exceptions +import time CONF = config.CONF @@ -48,3 +52,29 @@ class V21ComputeClient(rest_client.RestClient): resp, body = self.delete(uri) return self.get_resp(resp, body) + + def run_operation_obj(self, obj_type, obj_id, operation, attrs): + uri = '/{0}/{1}/{2}'.format(obj_type, obj_id, operation) + resp, body = self.post(uri, body=jsonutils.dumps(attrs)) + + return self.get_resp(resp, body) + + def get_obj(self, obj_type, obj_id, params=None): + uri = '/{0}/{1}'.format(obj_type, obj_id) + if params: + uri += '?{0}'.format(urllib.urlencode(params)) + resp, body = self.get(uri) + + return self.get_resp(resp, body) + + def wait_for_status(self, obj_type, obj_id, expected_status, timeout=None): + timeout = timeout or CONF.clustering.wait_timeout + + with timeutils.StopWatch(timeout) as timeout_watch: + while not timeout_watch.expired(): + time.sleep(5) + res = self.get_obj(obj_type, obj_id) + if res['body']['status'] == expected_status: + return res + + raise exceptions.TimeoutException() diff --git a/senlin_tempest_plugin/common/constants.py b/senlin_tempest_plugin/common/constants.py index 190451d..42c1726 100644 --- a/senlin_tempest_plugin/common/constants.py +++ b/senlin_tempest_plugin/common/constants.py @@ -24,7 +24,6 @@ spec_nova_server = { } } - spec_heat_stack = { "type": "os.heat.stack", "version": "1.0", @@ -54,7 +53,6 @@ spec_heat_stack = { } } - spec_scaling_policy = { "type": "senlin.policy.scaling", "version": "1.0", @@ -69,7 +67,6 @@ spec_scaling_policy = { } } - spec_lb_policy = { "type": "senlin.policy.loadbalance", "version": "1.1", @@ -104,7 +101,6 @@ spec_lb_policy = { } } - spec_batch_policy = { "type": "senlin.policy.batch", "version": "1.0", @@ -115,7 +111,6 @@ spec_batch_policy = { } } - spec_deletion_policy = { "type": "senlin.policy.deletion", "version": "1.1", @@ -124,7 +119,6 @@ spec_deletion_policy = { } } - spec_deletion_policy_with_hook = { "type": "senlin.policy.deletion", "version": "1.1", @@ -139,3 +133,85 @@ spec_deletion_policy_with_hook = { "criteria": "OLDEST_FIRST" } } + +spec_health_policy = { + "version": "1.1", + "type": "senlin.policy.health", + "description": "A policy for maintaining node health from a cluster.", + "properties": { + "detection": { + "detection_modes": [ + { + "type": "NODE_STATUS_POLLING" + }, + { + "type": "NODE_STATUS_POLL_URL", + "options": { + "poll_url_retry_limit": 3, + "poll_url": "http://127.0.0.1:5050", + "poll_url_retry_interval": 2 + } + } + ], + "node_update_timeout": 10, + "interval": 10 + }, + "recovery": { + "node_delete_timeout": 90, + "actions": [ + { + "name": "RECREATE" + } + ], + "node_force_recreate": True + } + } +} + +spec_health_policy_duplicate_type = { + "version": "1.1", + "type": "senlin.policy.health", + "properties": { + "detection": { + "detection_modes": [ + { + "type": "NODE_STATUS_POLLING" + }, + { + "type": "NODE_STATUS_POLLING", + } + ], + }, + "recovery": { + "actions": [ + { + "name": "RECREATE" + } + ], + } + } +} + +spec_health_policy_invalid_combo = { + "version": "1.1", + "type": "senlin.policy.health", + "properties": { + "detection": { + "detection_modes": [ + { + "type": "NODE_STATUS_POLLING" + }, + { + "type": "LIFECYCLE_EVENTS", + } + ], + }, + "recovery": { + "actions": [ + { + "name": "RECREATE" + } + ], + } + } +} diff --git a/senlin_tempest_plugin/common/network_client.py b/senlin_tempest_plugin/common/network_client.py index 071b890..4cc764b 100644 --- a/senlin_tempest_plugin/common/network_client.py +++ b/senlin_tempest_plugin/common/network_client.py @@ -13,8 +13,13 @@ # limitations under the License. from oslo_serialization import jsonutils +from oslo_utils import timeutils +import time + from tempest import config from tempest.lib.common import rest_client +from tempest.lib import exceptions + CONF = config.CONF @@ -45,8 +50,27 @@ class NetworkClient(rest_client.RestClient): return self.get_resp(resp, body) + def get_obj(self, obj_type, obj_id): + uri = '{0}/{1}/{2}'.format(self.uri_prefix, obj_type, obj_id) + resp, body = self.get(uri) + + return self.get_resp(resp, body) + def delete_obj(self, obj_type, obj_id): uri = '{0}/{1}/{2}'.format(self.uri_prefix, obj_type, obj_id) resp, body = self.delete(uri) return self.get_resp(resp, body) + + def wait_for_delete(self, obj_type, obj_id, timeout=None): + timeout = timeout or CONF.clustering.wait_timeout + + with timeutils.StopWatch(timeout) as timeout_watch: + while not timeout_watch.expired(): + try: + self.get_obj(obj_type, obj_id) + except exceptions.NotFound: + return + time.sleep(5) + + raise exceptions.TimeoutException() diff --git a/senlin_tempest_plugin/common/utils.py b/senlin_tempest_plugin/common/utils.py index 67d20d1..7376633 100644 --- a/senlin_tempest_plugin/common/utils.py +++ b/senlin_tempest_plugin/common/utils.py @@ -12,6 +12,7 @@ # limitations under the License. import functools +import subprocess from tempest.lib.common.utils import data_utils from tempest.lib import exceptions @@ -507,7 +508,8 @@ def create_a_network(base, name=None): return body['body']['id'] -def delete_a_network(base, network_id, ignore_missing=False): +def delete_a_network(base, network_id, ignore_missing=False, + wait_timeout=None): """Utility function that deletes a Neutron network.""" res = base.network_client.delete_obj('networks', network_id) @@ -516,6 +518,8 @@ def delete_a_network(base, network_id, ignore_missing=False): return raise exceptions.NotFound() + base.network_client.wait_for_delete('networks', network_id, wait_timeout) + def create_a_subnet(base, network_id, cidr, ip_version=4, name=None): """Utility function that creates a Neutron subnet""" @@ -536,7 +540,7 @@ def create_a_subnet(base, network_id, cidr, ip_version=4, name=None): return body['body']['id'] -def delete_a_subnet(base, subnet_id, ignore_missing=False): +def delete_a_subnet(base, subnet_id, ignore_missing=False, wait_timeout=None): """Utility function that deletes a Neutron subnet.""" res = base.network_client.delete_obj('subnets', subnet_id) @@ -545,6 +549,8 @@ def delete_a_subnet(base, subnet_id, ignore_missing=False): return raise exceptions.NotFound() + base.network_client.wait_for_delete('subnets', subnet_id, wait_timeout) + def create_queue(base, queue_name): """Utility function that creates Zaqar queue.""" @@ -582,3 +588,18 @@ def post_messages(base, queue_name, messages): if res['status'] != 201: msg = 'Failed in posting messages to Zaqar queue %s' % queue_name raise Exception(msg) + + +def start_http_server(port): + return subprocess.Popen( + ['python', '-m', 'SimpleHTTPServer', port], cwd='/tmp', + stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + + +def terminate_http_server(p): + if not p.poll(): + p.terminate() + return p.stdout.read() + + return diff --git a/senlin_tempest_plugin/pre_test_hook.sh b/senlin_tempest_plugin/pre_test_hook.sh index 788aa6f..496e32b 100755 --- a/senlin_tempest_plugin/pre_test_hook.sh +++ b/senlin_tempest_plugin/pre_test_hook.sh @@ -35,6 +35,7 @@ _LOG_CFG+=',oslo_messaging._drivers.amqpdriver=WARN' echo -e '[[post-config|$SENLIN_CONF]]\n[DEFAULT]\n' >> $localconf echo -e 'num_engine_workers=2\n' >> $localconf echo -e "cloud_backend=$SENLIN_BACKEND\n" >> $localconf +echo -e "health_check_interval_min=10\n" >> $localconf echo -e $_LOG_CFG >> $localconf if [[ "$SENLIN_BACKEND" == "openstack" ]]; then diff --git a/senlin_tempest_plugin/tests/functional/test_health_policy.py b/senlin_tempest_plugin/tests/functional/test_health_policy.py new file mode 100644 index 0000000..6c1a5c5 --- /dev/null +++ b/senlin_tempest_plugin/tests/functional/test_health_policy.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from tempest.lib import decorators +from tempest.lib import exceptions as exc +import time + +from senlin_tempest_plugin.common import constants +from senlin_tempest_plugin.common import utils +from senlin_tempest_plugin.tests.functional import base + + +class TestHealthPolicy(base.BaseSenlinFunctionalTest): + def setUp(self): + super(TestHealthPolicy, self).setUp() + + self.profile_id = utils.create_a_profile(self) + self.addCleanup(utils.delete_a_profile, self, self.profile_id) + self.cluster_id = utils.create_a_cluster(self, self.profile_id, + min_size=0, max_size=5, + desired_capacity=1) + self.addCleanup(utils.delete_a_cluster, self, self.cluster_id) + + @decorators.attr(type=['functional']) + @decorators.idempotent_id('adfd813c-08c4-4650-9b66-a1a6e63b842e') + def test_health_policy(self): + # Create a health policy + spec = constants.spec_health_policy + policy_id = utils.create_a_policy(self, spec) + del_policy = utils.get_a_policy(self, policy_id) + self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) + http_server = utils.start_http_server('5050') + self.addCleanup(utils.terminate_http_server, http_server) + + # Attach health policy to cluster + utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) + self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, + del_policy['id']) + + # wait for health checks to run + time.sleep(15) + + # check that URL was queried for each node as part of health check + out = utils.terminate_http_server(http_server) + self.assertTrue(out.count('GET') >= 1) + + @decorators.attr(type=['functional']) + @decorators.idempotent_id('569ca522-00ec-4c1e-b217-4f89d13fe800') + def test_invalid_health_policy_duplicate_type(self): + # Create a health policy + spec = constants.spec_health_policy_duplicate_type + with self.assertRaisesRegex( + exc.BadRequest, + '.*(?i)duplicate detection modes.*'): + utils.create_a_policy(self, spec) + + @decorators.attr(type=['functional']) + @decorators.idempotent_id('6f0e0d2c-4381-4afb-ac17-3c2cfed35829') + def test_invalid_health_policy_invalid_combo(self): + # Create a health policy + spec = constants.spec_health_policy_invalid_combo + with self.assertRaisesRegex( + exc.BadRequest, + '.*(?i)invalid detection modes.*'): + utils.create_a_policy(self, spec) diff --git a/senlin_tempest_plugin/tests/functional/test_lb_policy.py b/senlin_tempest_plugin/tests/functional/test_lb_policy.py index 2dee6b9..fd42f4a 100644 --- a/senlin_tempest_plugin/tests/functional/test_lb_policy.py +++ b/senlin_tempest_plugin/tests/functional/test_lb_policy.py @@ -29,7 +29,7 @@ class TestLBPolicy(base.BaseSenlinFunctionalTest): self.addCleanup(utils.delete_a_cluster, self, self.cluster_id) @decorators.attr(type=['functional']) - @decorators.idempotent_id('6b513a5d-75b6-447a-b95d-e17b84ac9ee8') + @decorators.idempotent_id('6b513a5d-75b6-447a-b95d-e17b84ac9ee2') def test_lb_policy(self): # Verify there is no lb information in node data cluster = utils.get_a_cluster(self, self.cluster_id) diff --git a/senlin_tempest_plugin/tests/integration/test_health_policy.py b/senlin_tempest_plugin/tests/integration/test_health_policy.py new file mode 100644 index 0000000..94b55e4 --- /dev/null +++ b/senlin_tempest_plugin/tests/integration/test_health_policy.py @@ -0,0 +1,188 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from tempest.lib import decorators +from tempest.lib import exceptions +import time + +from senlin_tempest_plugin.common import constants +from senlin_tempest_plugin.common import utils +from senlin_tempest_plugin.tests.integration import base + + +class TestHealthPolicy(base.BaseSenlinIntegrationTest): + def setUp(self): + super(TestHealthPolicy, self).setUp() + + self.profile_id = utils.create_a_profile(self) + self.addCleanup(utils.delete_a_profile, self, self.profile_id) + self.cluster_id = utils.create_a_cluster(self, self.profile_id, + min_size=0, max_size=5, + desired_capacity=1) + self.addCleanup(utils.delete_a_cluster, self, self.cluster_id) + + @decorators.attr(type=['integration']) + def test_health_policy(self): + # Create a health policy + spec = constants.spec_health_policy + policy_id = utils.create_a_policy(self, spec) + del_policy = utils.get_a_policy(self, policy_id) + self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) + http_server = utils.start_http_server('5050') + self.addCleanup(utils.terminate_http_server, http_server) + + # Attach health policy to cluster + utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) + self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, + del_policy['id']) + + # wait for health checks to run + time.sleep(5) + + # check that URL was queried for each node as part of health check + out = utils.terminate_http_server(http_server) + self.assertTrue(out.count('GET') == 1) + + def _get_node(self, expected_len, index): + # get physical id of node (cluster is expected to only contain 1 node) + raw_nodes = utils.list_nodes(self) + nodes = { + n['id']: n['physical_id'] for n in raw_nodes + if n['cluster_id'] == self.cluster_id + } + self.assertTrue(len(nodes) == expected_len) + + return nodes.keys()[index], nodes.values()[index] + + @decorators.attr(type=['integration']) + def test_multiple_detection_modes_any(self): + # Create a health policy + spec = constants.spec_health_policy + spec['properties']['detection']['recovery_conditional'] = 'ANY_FAILED' + policy_id = utils.create_a_policy(self, spec) + del_policy = utils.get_a_policy(self, policy_id) + self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) + http_server = utils.start_http_server('5050') + self.addCleanup(utils.terminate_http_server, http_server) + + # manually shutdown server + node_id, server_id = self._get_node(1, 0) + self.compute_client.run_operation_obj( + 'servers', server_id, 'action', {'os-stop': None}) + + # verify that server is shutdown + self.compute_client.wait_for_status('servers', server_id, + 'SHUTOFF', 60) + + # Attach health policy to cluster + utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) + self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, + del_policy['id']) + + # wait for health checks to run and recover node + time.sleep(15) + + # verify that node has been recovered + self.client.wait_for_status('nodes', node_id, 'ACTIVE', 60) + + # verify that new server is ACTIVE + old_server_id = server_id + node_id, server_id = self._get_node(1, 0) + self.assertNotEqual(old_server_id, server_id) + self.compute_client.wait_for_status('servers', server_id, 'ACTIVE', 60) + + # verify that old server no longer exists + self.assertRaises( + exceptions.NotFound, + self.compute_client.get_obj, 'servers', old_server_id) + + @decorators.attr(type=['integration']) + def test_multiple_detection_modes_all(self): + # Create a health policy + spec = constants.spec_health_policy + spec['properties']['detection']['recovery_conditional'] = 'ALL_FAILED' + policy_id = utils.create_a_policy(self, spec) + del_policy = utils.get_a_policy(self, policy_id) + self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) + http_server = utils.start_http_server('5050') + self.addCleanup(utils.terminate_http_server, http_server) + + # manually shutdown server + node_id, server_id = self._get_node(1, 0) + self.compute_client.run_operation_obj( + 'servers', server_id, 'action', {'os-stop': None}) + + # verify that server is shutdown + self.compute_client.wait_for_status('servers', server_id, + 'SHUTOFF', 60) + + # Attach health policy to cluster + utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) + self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, + del_policy['id']) + + # wait for health checks to run + time.sleep(15) + + # verify that node status has not changed + res = self.client.get_obj('nodes', node_id) + self.assertEqual(res['body']['status'], 'ACTIVE') + + # verify that server is still stopped + res = self.compute_client.get_obj('servers', server_id) + self.assertEqual(res['body']['status'], 'SHUTOFF') + + # check that URL was queried because ALL_FAILED + # was specified in the policy + out = utils.terminate_http_server(http_server) + self.assertTrue(out.count('GET') >= 0) + + # wait for health checks to run and recover node + time.sleep(15) + + # verify that node has been recovered + self.client.wait_for_status('nodes', node_id, 'ACTIVE', 60) + + # verify that new server is ACTIVE + old_server_id = server_id + node_id, server_id = self._get_node(1, 0) + self.assertNotEqual(old_server_id, server_id) + self.compute_client.wait_for_status('servers', server_id, 'ACTIVE', 60) + + # verify that old server no longer exists + self.assertRaises( + exceptions.NotFound, + self.compute_client.get_obj, 'servers', old_server_id) + + @decorators.attr(type=['integration']) + def test_multiple_detection_modes_all_poll_url_fail(self): + # Create a health policy + spec = constants.spec_health_policy + spec['properties']['detection']['recovery_conditional'] = 'ALL_FAILED' + policy_id = utils.create_a_policy(self, spec) + del_policy = utils.get_a_policy(self, policy_id) + self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) + + # get node_id + node_id, server_id = self._get_node(1, 0) + + # Attach health policy to cluster without http server running + utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) + self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, + del_policy['id']) + + # wait for health checks to run + time.sleep(15) + + # verify that node status has not changed + res = self.client.get_obj('nodes', node_id) + self.assertEqual(res['body']['status'], 'ACTIVE')