Fix health policy tests
- Change clustering client delete and trigger_action to use request method instead of delete or post directly. This way exceptions will be propagated. - Modified clustering client wait_for_status to accept list of expected statuses. - Use tenacity to retry delete cluster and detach policy calls if it returns Conflict because of locked cluster. - Wait for action status to become SUCCEEDED or FAILED when calling delete cluster or detach policy. - Use custom detach policy as cleanup action so that it waits for 5 seconds to allow the health manager to stop health check threads - Use separate prepare_and_cleanup_nova_server function call for nova_server_cluster test and health_policy tests to avoid race condition between the tests. - Make list of dictionary keys before indexing (need for python 3) Depends-On: https://review.openstack.org/#/c/624248/ Closes-Bug: 1797270 Change-Id: If34be5d01fc9e58d548f68e85b691d3ed7bd4c51
This commit is contained in:
Duc Truong
parent
c7fbb42f17
commit
f61b01f746
@ -10,3 +10,4 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
|
||||
oslo.utils>=3.33.0 # Apache-2.0
|
||||
six>=1.10.0 # MIT
|
||||
tempest>=17.1.0 # Apache-2.0
|
||||
tenacity>=4.9.0 # Apache-2.0
|
||||
|
||||
@ -84,7 +84,7 @@ class ClusteringAPIClient(rest_client.RestClient):
|
||||
|
||||
def delete_obj(self, obj_type, obj_id):
|
||||
uri = '{0}/{1}/{2}'.format(self.version, obj_type, obj_id)
|
||||
resp, body = self.delete(uri)
|
||||
resp, body = self.request('DELETE', uri)
|
||||
|
||||
return self.get_resp(resp, body)
|
||||
|
||||
@ -106,7 +106,7 @@ class ClusteringAPIClient(rest_client.RestClient):
|
||||
uri = '{0}/{1}/{2}/actions'.format(self.version, obj_type, obj_id)
|
||||
if params is not None:
|
||||
params = jsonutils.dumps(params)
|
||||
resp, body = self.post(uri, body=params)
|
||||
resp, body = self.request('POST', uri, body=params)
|
||||
|
||||
return self.get_resp(resp, body)
|
||||
|
||||
@ -148,6 +148,11 @@ class ClusteringAPIClient(rest_client.RestClient):
|
||||
return self.get_resp(resp, body)
|
||||
|
||||
def wait_for_status(self, obj_type, obj_id, expected_status, timeout=None):
|
||||
if isinstance(expected_status, list):
|
||||
expected_status_list = expected_status
|
||||
else:
|
||||
expected_status_list = [expected_status]
|
||||
|
||||
if timeout is None:
|
||||
timeout = CONF.clustering.wait_timeout
|
||||
|
||||
@ -155,7 +160,7 @@ class ClusteringAPIClient(rest_client.RestClient):
|
||||
while timeout > 0:
|
||||
time.sleep(5)
|
||||
res = self.get_obj(obj_type, obj_id)
|
||||
if res['body']['status'] == expected_status:
|
||||
if res['body']['status'] in expected_status_list:
|
||||
return res
|
||||
timeout = timeout_watch.leftover(True)
|
||||
|
||||
|
||||
@ -12,7 +12,12 @@
|
||||
# limitations under the License.
|
||||
|
||||
import functools
|
||||
import subprocess
|
||||
import os
|
||||
import signal
|
||||
from six.moves import BaseHTTPServer
|
||||
from six.moves import http_client as http
|
||||
import tempfile
|
||||
import tenacity
|
||||
|
||||
from tempest.lib.common.utils import data_utils
|
||||
from tempest.lib import exceptions
|
||||
@ -33,9 +38,13 @@ def api_microversion(api_microversion):
|
||||
return decorator
|
||||
|
||||
|
||||
def prepare_and_cleanup_for_nova_server(base):
|
||||
def prepare_and_cleanup_for_nova_server(base, cidr, spec=None):
|
||||
keypair_name = create_a_keypair(base, is_admin_manager=False)
|
||||
base.spec = constants.spec_nova_server
|
||||
if spec is None:
|
||||
base.spec = constants.spec_nova_server
|
||||
else:
|
||||
base.spec = spec
|
||||
|
||||
base.spec['properties']['key_name'] = keypair_name
|
||||
base.addCleanup(delete_a_keypair, base, keypair_name,
|
||||
is_admin_manager=False)
|
||||
@ -44,7 +53,7 @@ def prepare_and_cleanup_for_nova_server(base):
|
||||
network_id = create_a_network(base, name=n_name)
|
||||
base.addCleanup(delete_a_network, base, network_id)
|
||||
|
||||
subnet_id = create_a_subnet(base, network_id, "192.168.199.0/24")
|
||||
subnet_id = create_a_subnet(base, network_id, cidr)
|
||||
base.addCleanup(delete_a_subnet, base, subnet_id)
|
||||
|
||||
|
||||
@ -147,14 +156,29 @@ def list_clusters(base):
|
||||
return res['body']
|
||||
|
||||
|
||||
def _return_last_value(retry_state):
|
||||
return retry_state.outcome.result()
|
||||
|
||||
|
||||
@tenacity.retry(
|
||||
retry=(tenacity.retry_if_exception_type(exceptions.Conflict) |
|
||||
tenacity.retry_if_result(lambda x: x is False)),
|
||||
wait=tenacity.wait_fixed(2),
|
||||
retry_error_callback=_return_last_value,
|
||||
stop=tenacity.stop_after_attempt(5)
|
||||
)
|
||||
def delete_a_cluster(base, cluster_id, wait_timeout=None):
|
||||
"""Utility function that deletes a Senlin cluster."""
|
||||
res = base.client.delete_obj('clusters', cluster_id)
|
||||
action_id = res['location'].split('/actions/')[1]
|
||||
base.client.wait_for_status('actions', action_id, 'SUCCEEDED',
|
||||
wait_timeout)
|
||||
|
||||
action = base.client.wait_for_status(
|
||||
'actions', action_id, ['SUCCEEDED', 'FAILED'], wait_timeout)
|
||||
if action['body']['status'] == 'FAILED':
|
||||
return False
|
||||
|
||||
base.client.wait_for_delete("clusters", cluster_id, wait_timeout)
|
||||
return
|
||||
return True
|
||||
|
||||
|
||||
def create_a_node(base, profile_id, cluster_id=None, metadata=None,
|
||||
@ -285,6 +309,13 @@ def cluster_attach_policy(base, cluster_id, policy_id,
|
||||
return res['body']['status_reason']
|
||||
|
||||
|
||||
@tenacity.retry(
|
||||
retry=(tenacity.retry_if_exception_type(exceptions.Conflict) |
|
||||
tenacity.retry_if_result(lambda x: x is False)),
|
||||
wait=tenacity.wait_fixed(2),
|
||||
retry_error_callback=_return_last_value,
|
||||
stop=tenacity.stop_after_attempt(5)
|
||||
)
|
||||
def cluster_detach_policy(base, cluster_id, policy_id,
|
||||
expected_status='SUCCEEDED', wait_timeout=None):
|
||||
"""Utility function that detach a policy from cluster."""
|
||||
@ -296,8 +327,12 @@ def cluster_detach_policy(base, cluster_id, policy_id,
|
||||
}
|
||||
res = base.client.trigger_action('clusters', cluster_id, params=params)
|
||||
action_id = res['location'].split('/actions/')[1]
|
||||
res = base.client.wait_for_status('actions', action_id, expected_status,
|
||||
wait_timeout)
|
||||
|
||||
res = base.client.wait_for_status(
|
||||
'actions', action_id, ['SUCCEEDED', 'FAILED'], wait_timeout)
|
||||
if res['body']['status'] == 'FAILED':
|
||||
return False
|
||||
|
||||
return res['body']['status_reason']
|
||||
|
||||
|
||||
@ -590,16 +625,46 @@ def post_messages(base, queue_name, messages):
|
||||
raise Exception(msg)
|
||||
|
||||
|
||||
def start_http_server(port):
|
||||
return subprocess.Popen(
|
||||
['python', '-m', 'SimpleHTTPServer', port], cwd='/tmp',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
def start_http_server(port=5050):
|
||||
def _get_http_handler_class(filename):
|
||||
class StaticHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
|
||||
def do_GET(self):
|
||||
data = b'healthy\n'
|
||||
self.send_response(http.OK)
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
return
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
msg = fmt % args
|
||||
with open(filename, 'a+') as tmp:
|
||||
tmp.write("%s\n" % msg)
|
||||
return
|
||||
|
||||
return StaticHTTPRequestHandler
|
||||
|
||||
server_address = ('127.0.0.1', port)
|
||||
new_file, filename = tempfile.mkstemp()
|
||||
handler_class = _get_http_handler_class(filename)
|
||||
httpd = BaseHTTPServer.HTTPServer(server_address, handler_class)
|
||||
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
httpd.serve_forever()
|
||||
else:
|
||||
return pid, filename
|
||||
|
||||
|
||||
def terminate_http_server(p):
|
||||
if not p.poll():
|
||||
p.terminate()
|
||||
return p.stdout.read()
|
||||
def terminate_http_server(pid, filename):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
|
||||
return
|
||||
if not os.path.isfile(filename):
|
||||
return ''
|
||||
|
||||
with open(filename, 'r') as f:
|
||||
contents = f.read()
|
||||
|
||||
os.remove(filename)
|
||||
return contents
|
||||
|
||||
@ -19,7 +19,6 @@ from senlin_tempest_plugin.common import utils
|
||||
from senlin_tempest_plugin.tests.functional import base
|
||||
|
||||
|
||||
@decorators.skip_because(bug=1797270)
|
||||
class TestHealthPolicy(base.BaseSenlinFunctionalTest):
|
||||
def setUp(self):
|
||||
super(TestHealthPolicy, self).setUp()
|
||||
@ -39,19 +38,30 @@ class TestHealthPolicy(base.BaseSenlinFunctionalTest):
|
||||
policy_id = utils.create_a_policy(self, spec)
|
||||
del_policy = utils.get_a_policy(self, policy_id)
|
||||
self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True)
|
||||
http_server = utils.start_http_server('5050')
|
||||
self.addCleanup(utils.terminate_http_server, http_server)
|
||||
http_server, log_file = utils.start_http_server()
|
||||
self.addCleanup(utils.terminate_http_server, http_server, log_file)
|
||||
|
||||
def detach_policy():
|
||||
# ignore BadRequest exceptions that are raised because
|
||||
# policy is not attached
|
||||
try:
|
||||
utils.cluster_detach_policy(self, self.cluster_id,
|
||||
del_policy['id'])
|
||||
|
||||
# wait to let health checks stop
|
||||
time.sleep(5)
|
||||
except exc.BadRequest:
|
||||
pass
|
||||
|
||||
# Attach health policy to cluster
|
||||
utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
|
||||
self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
|
||||
del_policy['id'])
|
||||
self.addCleanup(detach_policy)
|
||||
|
||||
# wait for health checks to run
|
||||
time.sleep(15)
|
||||
|
||||
# check that URL was queried for each node as part of health check
|
||||
out = utils.terminate_http_server(http_server)
|
||||
out = utils.terminate_http_server(http_server, log_file)
|
||||
self.assertTrue(out.count('GET') >= 1)
|
||||
|
||||
@decorators.attr(type=['functional'])
|
||||
|
||||
@ -19,11 +19,14 @@ from senlin_tempest_plugin.common import utils
|
||||
from senlin_tempest_plugin.tests.integration import base
|
||||
|
||||
|
||||
@decorators.skip_because(bug=1797270)
|
||||
class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
def setUp(self):
|
||||
super(TestHealthPolicy, self).setUp()
|
||||
|
||||
spec = constants.spec_nova_server
|
||||
spec['properties']['networks'][0]['network'] = 'private-hp'
|
||||
utils.prepare_and_cleanup_for_nova_server(self, "192.168.199.0/24",
|
||||
spec)
|
||||
self.profile_id = utils.create_a_profile(self)
|
||||
self.addCleanup(utils.delete_a_profile, self, self.profile_id)
|
||||
self.cluster_id = utils.create_a_cluster(self, self.profile_id,
|
||||
@ -31,6 +34,17 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
desired_capacity=1)
|
||||
self.addCleanup(utils.delete_a_cluster, self, self.cluster_id)
|
||||
|
||||
def _detach_policy(self, policy_id):
|
||||
# ignore BadRequest exceptions that are raised because
|
||||
# policy is not attached
|
||||
try:
|
||||
utils.cluster_detach_policy(self, self.cluster_id, policy_id)
|
||||
|
||||
# wait to let health checks stop
|
||||
time.sleep(5)
|
||||
except exceptions.BadRequest:
|
||||
pass
|
||||
|
||||
@decorators.attr(type=['integration'])
|
||||
def test_health_policy(self):
|
||||
# Create a health policy
|
||||
@ -38,19 +52,18 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
policy_id = utils.create_a_policy(self, spec)
|
||||
del_policy = utils.get_a_policy(self, policy_id)
|
||||
self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True)
|
||||
http_server = utils.start_http_server('5050')
|
||||
self.addCleanup(utils.terminate_http_server, http_server)
|
||||
http_server, log_file = utils.start_http_server()
|
||||
self.addCleanup(utils.terminate_http_server, http_server, log_file)
|
||||
|
||||
# Attach health policy to cluster
|
||||
utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
|
||||
self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
|
||||
del_policy['id'])
|
||||
self.addCleanup(self._detach_policy, del_policy['id'])
|
||||
|
||||
# wait for health checks to run
|
||||
time.sleep(5)
|
||||
|
||||
# check that URL was queried for each node as part of health check
|
||||
out = utils.terminate_http_server(http_server)
|
||||
out = utils.terminate_http_server(http_server, log_file)
|
||||
self.assertTrue(out.count('GET') == 1)
|
||||
|
||||
def _get_node(self, expected_len, index):
|
||||
@ -62,7 +75,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
}
|
||||
self.assertTrue(len(nodes) == expected_len)
|
||||
|
||||
return nodes.keys()[index], nodes.values()[index]
|
||||
return list(nodes.keys())[index], list(nodes.values())[index]
|
||||
|
||||
@decorators.attr(type=['integration'])
|
||||
def test_multiple_detection_modes_any(self):
|
||||
@ -72,8 +85,8 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
policy_id = utils.create_a_policy(self, spec)
|
||||
del_policy = utils.get_a_policy(self, policy_id)
|
||||
self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True)
|
||||
http_server = utils.start_http_server('5050')
|
||||
self.addCleanup(utils.terminate_http_server, http_server)
|
||||
http_server, log_file = utils.start_http_server()
|
||||
self.addCleanup(utils.terminate_http_server, http_server, log_file)
|
||||
|
||||
# manually shutdown server
|
||||
node_id, server_id = self._get_node(1, 0)
|
||||
@ -86,8 +99,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
|
||||
# Attach health policy to cluster
|
||||
utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
|
||||
self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
|
||||
del_policy['id'])
|
||||
self.addCleanup(self._detach_policy, del_policy['id'])
|
||||
|
||||
# wait for health checks to run and recover node
|
||||
time.sleep(15)
|
||||
@ -114,8 +126,8 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
policy_id = utils.create_a_policy(self, spec)
|
||||
del_policy = utils.get_a_policy(self, policy_id)
|
||||
self.addCleanup(utils.delete_a_policy, self, del_policy['id'], True)
|
||||
http_server = utils.start_http_server('5050')
|
||||
self.addCleanup(utils.terminate_http_server, http_server)
|
||||
http_server, log_file = utils.start_http_server()
|
||||
self.addCleanup(utils.terminate_http_server, http_server, log_file)
|
||||
|
||||
# manually shutdown server
|
||||
node_id, server_id = self._get_node(1, 0)
|
||||
@ -128,8 +140,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
|
||||
# Attach health policy to cluster
|
||||
utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
|
||||
self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
|
||||
del_policy['id'])
|
||||
self.addCleanup(self._detach_policy, del_policy['id'])
|
||||
|
||||
# wait for health checks to run
|
||||
time.sleep(15)
|
||||
@ -144,7 +155,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
|
||||
# check that URL was queried because ALL_FAILED
|
||||
# was specified in the policy
|
||||
out = utils.terminate_http_server(http_server)
|
||||
out = utils.terminate_http_server(http_server, log_file)
|
||||
self.assertTrue(out.count('GET') >= 0)
|
||||
|
||||
# wait for health checks to run and recover node
|
||||
@ -178,8 +189,7 @@ class TestHealthPolicy(base.BaseSenlinIntegrationTest):
|
||||
|
||||
# Attach health policy to cluster without http server running
|
||||
utils.cluster_attach_policy(self, self.cluster_id, del_policy['id'])
|
||||
self.addCleanup(utils.cluster_detach_policy, self, self.cluster_id,
|
||||
del_policy['id'])
|
||||
self.addCleanup(self._detach_policy, del_policy['id'])
|
||||
|
||||
# wait for health checks to run
|
||||
time.sleep(15)
|
||||
|
||||
@ -23,7 +23,7 @@ class TestNovaServerCluster(base.BaseSenlinIntegrationNonAdminTest):
|
||||
|
||||
def setUp(self):
|
||||
super(TestNovaServerCluster, self).setUp()
|
||||
utils.prepare_and_cleanup_for_nova_server(self)
|
||||
utils.prepare_and_cleanup_for_nova_server(self, "192.168.199.0/24")
|
||||
self.profile_id = utils.create_a_profile(self, self.spec)
|
||||
self.addCleanup(utils.delete_a_profile, self, self.profile_id)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user