diff --git a/requirements.txt b/requirements.txt index d44710b..487388f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 oslo.utils>=3.33.0 # Apache-2.0 six>=1.10.0 # MIT tempest>=17.1.0 # Apache-2.0 +tenacity>=4.9.0 # Apache-2.0 diff --git a/senlin_tempest_plugin/common/clustering_client.py b/senlin_tempest_plugin/common/clustering_client.py index 79a3099..d5c98f1 100644 --- a/senlin_tempest_plugin/common/clustering_client.py +++ b/senlin_tempest_plugin/common/clustering_client.py @@ -84,7 +84,7 @@ class ClusteringAPIClient(rest_client.RestClient): def delete_obj(self, obj_type, obj_id): uri = '{0}/{1}/{2}'.format(self.version, obj_type, obj_id) - resp, body = self.delete(uri) + resp, body = self.request('DELETE', uri) return self.get_resp(resp, body) @@ -106,7 +106,7 @@ class ClusteringAPIClient(rest_client.RestClient): uri = '{0}/{1}/{2}/actions'.format(self.version, obj_type, obj_id) if params is not None: params = jsonutils.dumps(params) - resp, body = self.post(uri, body=params) + resp, body = self.request('POST', uri, body=params) return self.get_resp(resp, body) @@ -148,6 +148,11 @@ class ClusteringAPIClient(rest_client.RestClient): return self.get_resp(resp, body) def wait_for_status(self, obj_type, obj_id, expected_status, timeout=None): + if isinstance(expected_status, list): + expected_status_list = expected_status + else: + expected_status_list = [expected_status] + if timeout is None: timeout = CONF.clustering.wait_timeout @@ -155,7 +160,7 @@ class ClusteringAPIClient(rest_client.RestClient): while timeout > 0: time.sleep(5) res = self.get_obj(obj_type, obj_id) - if res['body']['status'] == expected_status: + if res['body']['status'] in expected_status_list: return res timeout = timeout_watch.leftover(True) diff --git a/senlin_tempest_plugin/common/utils.py b/senlin_tempest_plugin/common/utils.py index 7376633..692e1d2 100644 --- a/senlin_tempest_plugin/common/utils.py +++ b/senlin_tempest_plugin/common/utils.py @@ -12,7 +12,12 @@ # limitations under the License. import functools -import subprocess +import os +import signal +from six.moves import BaseHTTPServer +from six.moves import http_client as http +import tempfile +import tenacity from tempest.lib.common.utils import data_utils from tempest.lib import exceptions @@ -33,9 +38,13 @@ def api_microversion(api_microversion): return decorator -def prepare_and_cleanup_for_nova_server(base): +def prepare_and_cleanup_for_nova_server(base, cidr, spec=None): keypair_name = create_a_keypair(base, is_admin_manager=False) - base.spec = constants.spec_nova_server + if spec is None: + base.spec = constants.spec_nova_server + else: + base.spec = spec + base.spec['properties']['key_name'] = keypair_name base.addCleanup(delete_a_keypair, base, keypair_name, is_admin_manager=False) @@ -44,7 +53,7 @@ def prepare_and_cleanup_for_nova_server(base): network_id = create_a_network(base, name=n_name) base.addCleanup(delete_a_network, base, network_id) - subnet_id = create_a_subnet(base, network_id, "192.168.199.0/24") + subnet_id = create_a_subnet(base, network_id, cidr) base.addCleanup(delete_a_subnet, base, subnet_id) @@ -147,14 +156,29 @@ def list_clusters(base): return res['body'] +def _return_last_value(retry_state): + return retry_state.outcome.result() + + +@tenacity.retry( + retry=(tenacity.retry_if_exception_type(exceptions.Conflict) | + tenacity.retry_if_result(lambda x: x is False)), + wait=tenacity.wait_fixed(2), + retry_error_callback=_return_last_value, + stop=tenacity.stop_after_attempt(5) +) def delete_a_cluster(base, cluster_id, wait_timeout=None): """Utility function that deletes a Senlin cluster.""" res = base.client.delete_obj('clusters', cluster_id) action_id = res['location'].split('/actions/')[1] - base.client.wait_for_status('actions', action_id, 'SUCCEEDED', - wait_timeout) + + action = base.client.wait_for_status( + 'actions', action_id, ['SUCCEEDED', 'FAILED'], wait_timeout) + if action['body']['status'] == 'FAILED': + return False + base.client.wait_for_delete("clusters", cluster_id, wait_timeout) - return + return True def create_a_node(base, profile_id, cluster_id=None, metadata=None, @@ -285,6 +309,13 @@ def cluster_attach_policy(base, cluster_id, policy_id, return res['body']['status_reason'] +@tenacity.retry( + retry=(tenacity.retry_if_exception_type(exceptions.Conflict) | + tenacity.retry_if_result(lambda x: x is False)), + wait=tenacity.wait_fixed(2), + retry_error_callback=_return_last_value, + stop=tenacity.stop_after_attempt(5) +) def cluster_detach_policy(base, cluster_id, policy_id, expected_status='SUCCEEDED', wait_timeout=None): """Utility function that detach a policy from cluster.""" @@ -296,8 +327,12 @@ def cluster_detach_policy(base, cluster_id, policy_id, } res = base.client.trigger_action('clusters', cluster_id, params=params) action_id = res['location'].split('/actions/')[1] - res = base.client.wait_for_status('actions', action_id, expected_status, - wait_timeout) + + res = base.client.wait_for_status( + 'actions', action_id, ['SUCCEEDED', 'FAILED'], wait_timeout) + if res['body']['status'] == 'FAILED': + return False + return res['body']['status_reason'] @@ -590,16 +625,46 @@ def post_messages(base, queue_name, messages): raise Exception(msg) -def start_http_server(port): - return subprocess.Popen( - ['python', '-m', 'SimpleHTTPServer', port], cwd='/tmp', - stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) +def start_http_server(port=5050): + def _get_http_handler_class(filename): + class StaticHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + + def do_GET(self): + data = b'healthy\n' + self.send_response(http.OK) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + return + + def log_message(self, fmt, *args): + msg = fmt % args + with open(filename, 'a+') as tmp: + tmp.write("%s\n" % msg) + return + + return StaticHTTPRequestHandler + + server_address = ('127.0.0.1', port) + new_file, filename = tempfile.mkstemp() + handler_class = _get_http_handler_class(filename) + httpd = BaseHTTPServer.HTTPServer(server_address, handler_class) + + pid = os.fork() + if pid == 0: + httpd.serve_forever() + else: + return pid, filename -def terminate_http_server(p): - if not p.poll(): - p.terminate() - return p.stdout.read() +def terminate_http_server(pid, filename): + os.kill(pid, signal.SIGKILL) - return + if not os.path.isfile(filename): + return '' + + with open(filename, 'r') as f: + contents = f.read() + + os.remove(filename) + return contents diff --git a/senlin_tempest_plugin/tests/functional/test_health_policy.py b/senlin_tempest_plugin/tests/functional/test_health_policy.py index 1d56c15..9200066 100644 --- a/senlin_tempest_plugin/tests/functional/test_health_policy.py +++ b/senlin_tempest_plugin/tests/functional/test_health_policy.py @@ -19,7 +19,6 @@ from senlin_tempest_plugin.common import utils from senlin_tempest_plugin.tests.functional import base -@decorators.skip_because(bug=1797270) class TestHealthPolicy(base.BaseSenlinFunctionalTest): def setUp(self): super(TestHealthPolicy, self).setUp() @@ -39,19 +38,30 @@ class TestHealthPolicy(base.BaseSenlinFunctionalTest): policy_id = utils.create_a_policy(self, spec) del_policy = utils.get_a_policy(self, policy_id) self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) - http_server = utils.start_http_server('5050') - self.addCleanup(utils.terminate_http_server, http_server) + http_server, log_file = utils.start_http_server() + self.addCleanup(utils.terminate_http_server, http_server, log_file) + + def detach_policy(): + # ignore BadRequest exceptions that are raised because + # policy is not attached + try: + utils.cluster_detach_policy(self, self.cluster_id, + del_policy['id']) + + # wait to let health checks stop + time.sleep(5) + except exc.BadRequest: + pass # Attach health policy to cluster utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) - self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, - del_policy['id']) + self.addCleanup(detach_policy) # wait for health checks to run time.sleep(15) # check that URL was queried for each node as part of health check - out = utils.terminate_http_server(http_server) + out = utils.terminate_http_server(http_server, log_file) self.assertTrue(out.count('GET') >= 1) @decorators.attr(type=['functional']) diff --git a/senlin_tempest_plugin/tests/integration/test_health_policy.py b/senlin_tempest_plugin/tests/integration/test_health_policy.py index fe951aa..0954f63 100644 --- a/senlin_tempest_plugin/tests/integration/test_health_policy.py +++ b/senlin_tempest_plugin/tests/integration/test_health_policy.py @@ -19,11 +19,14 @@ from senlin_tempest_plugin.common import utils from senlin_tempest_plugin.tests.integration import base -@decorators.skip_because(bug=1797270) class TestHealthPolicy(base.BaseSenlinIntegrationTest): def setUp(self): super(TestHealthPolicy, self).setUp() + spec = constants.spec_nova_server + spec['properties']['networks'][0]['network'] = 'private-hp' + utils.prepare_and_cleanup_for_nova_server(self, "192.168.199.0/24", + spec) self.profile_id = utils.create_a_profile(self) self.addCleanup(utils.delete_a_profile, self, self.profile_id) self.cluster_id = utils.create_a_cluster(self, self.profile_id, @@ -31,6 +34,17 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): desired_capacity=1) self.addCleanup(utils.delete_a_cluster, self, self.cluster_id) + def _detach_policy(self, policy_id): + # ignore BadRequest exceptions that are raised because + # policy is not attached + try: + utils.cluster_detach_policy(self, self.cluster_id, policy_id) + + # wait to let health checks stop + time.sleep(5) + except exceptions.BadRequest: + pass + @decorators.attr(type=['integration']) def test_health_policy(self): # Create a health policy @@ -38,19 +52,18 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): policy_id = utils.create_a_policy(self, spec) del_policy = utils.get_a_policy(self, policy_id) self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) - http_server = utils.start_http_server('5050') - self.addCleanup(utils.terminate_http_server, http_server) + http_server, log_file = utils.start_http_server() + self.addCleanup(utils.terminate_http_server, http_server, log_file) # Attach health policy to cluster utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) - self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, - del_policy['id']) + self.addCleanup(self._detach_policy, del_policy['id']) # wait for health checks to run time.sleep(5) # check that URL was queried for each node as part of health check - out = utils.terminate_http_server(http_server) + out = utils.terminate_http_server(http_server, log_file) self.assertTrue(out.count('GET') == 1) def _get_node(self, expected_len, index): @@ -62,7 +75,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): } self.assertTrue(len(nodes) == expected_len) - return nodes.keys()[index], nodes.values()[index] + return list(nodes.keys())[index], list(nodes.values())[index] @decorators.attr(type=['integration']) def test_multiple_detection_modes_any(self): @@ -72,8 +85,8 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): policy_id = utils.create_a_policy(self, spec) del_policy = utils.get_a_policy(self, policy_id) self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) - http_server = utils.start_http_server('5050') - self.addCleanup(utils.terminate_http_server, http_server) + http_server, log_file = utils.start_http_server() + self.addCleanup(utils.terminate_http_server, http_server, log_file) # manually shutdown server node_id, server_id = self._get_node(1, 0) @@ -86,8 +99,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): # Attach health policy to cluster utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) - self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, - del_policy['id']) + self.addCleanup(self._detach_policy, del_policy['id']) # wait for health checks to run and recover node time.sleep(15) @@ -114,8 +126,8 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): policy_id = utils.create_a_policy(self, spec) del_policy = utils.get_a_policy(self, policy_id) self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True) - http_server = utils.start_http_server('5050') - self.addCleanup(utils.terminate_http_server, http_server) + http_server, log_file = utils.start_http_server() + self.addCleanup(utils.terminate_http_server, http_server, log_file) # manually shutdown server node_id, server_id = self._get_node(1, 0) @@ -128,8 +140,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): # Attach health policy to cluster utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) - self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, - del_policy['id']) + self.addCleanup(self._detach_policy, del_policy['id']) # wait for health checks to run time.sleep(15) @@ -144,7 +155,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): # check that URL was queried because ALL_FAILED # was specified in the policy - out = utils.terminate_http_server(http_server) + out = utils.terminate_http_server(http_server, log_file) self.assertTrue(out.count('GET') >= 0) # wait for health checks to run and recover node @@ -178,8 +189,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest): # Attach health policy to cluster without http server running utils.cluster_attach_policy(self, self.cluster_id, del_policy['id']) - self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id, - del_policy['id']) + self.addCleanup(self._detach_policy, del_policy['id']) # wait for health checks to run time.sleep(15) diff --git a/senlin_tempest_plugin/tests/integration/test_nova_server_cluster.py b/senlin_tempest_plugin/tests/integration/test_nova_server_cluster.py index c3fec81..f1d902f 100644 --- a/senlin_tempest_plugin/tests/integration/test_nova_server_cluster.py +++ b/senlin_tempest_plugin/tests/integration/test_nova_server_cluster.py @@ -23,7 +23,7 @@ class TestNovaServerCluster(base.BaseSenlinIntegrationNonAdminTest): def setUp(self): super(TestNovaServerCluster, self).setUp() - utils.prepare_and_cleanup_for_nova_server(self) + utils.prepare_and_cleanup_for_nova_server(self, "192.168.199.0/24") self.profile_id = utils.create_a_profile(self, self.spec) self.addCleanup(utils.delete_a_profile, self, self.profile_id)