Add test_service_with_not_ready_endpoints test
This test case automates the issue described in BZ https://bugzilla.redhat.com/show_bug.cgi?id=1980957. The new test case: - creates a deployment with a failing readiness probe so all the pods are not ready - creates a service for the deployment (where the endpoints are not ready) - it checks Kuryr pods are not restarted Change-Id: Id19f85625901b35900f9441aace8fe8938cb1d68
This commit is contained in:
parent
d9a93bcba9
commit
2ffdd2f475
@ -284,13 +284,21 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_pods_ready_num(cls, namespace="default",
|
def check_pods_ready_num(cls, namespace="default",
|
||||||
label="", num_pods=1):
|
label="", num_pods=1):
|
||||||
pods = cls.get_pod_name_list(namespace=namespace,
|
pods = cls.get_pod_name_list(namespace=namespace,
|
||||||
label_selector=label)
|
label_selector=label)
|
||||||
ready_pods = sum([cls.get_readiness_state(p) for p in pods])
|
ready_pods = sum([cls.get_readiness_state(p) for p in pods])
|
||||||
return (num_pods == ready_pods)
|
return (num_pods == ready_pods)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def check_pods_status_num(cls, namespace="default", label="", num_pods=1,
|
||||||
|
status="Running"):
|
||||||
|
pods = cls.get_pod_name_list(namespace=namespace,
|
||||||
|
label_selector=label)
|
||||||
|
status_pods = sum([cls.get_pod_status(p) == status for p in pods])
|
||||||
|
return (num_pods == status_pods)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_pod_readiness(cls, pod_name, namespace="default"):
|
def get_pod_readiness(cls, pod_name, namespace="default"):
|
||||||
LOG.info("Checking if pod {} is ready".format(pod_name))
|
LOG.info("Checking if pod {} is ready".format(pod_name))
|
||||||
@ -406,7 +414,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||||||
|
|
||||||
def create_deployment(self, deployment_name=None, api_version="apps/v1",
|
def create_deployment(self, deployment_name=None, api_version="apps/v1",
|
||||||
kind="Deployment", namespace="default",
|
kind="Deployment", namespace="default",
|
||||||
labels={"app": "demo"}):
|
labels={"app": "demo"}, failing_probe=False):
|
||||||
api_instance = kubernetes.client.AppsV1Api()
|
api_instance = kubernetes.client.AppsV1Api()
|
||||||
if not deployment_name:
|
if not deployment_name:
|
||||||
deployment_name = data_utils.rand_name(prefix='kuryr-deployment')
|
deployment_name = data_utils.rand_name(prefix='kuryr-deployment')
|
||||||
@ -418,6 +426,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||||||
{"image": "quay.io/kuryr/demo",
|
{"image": "quay.io/kuryr/demo",
|
||||||
"name": 'demo',
|
"name": 'demo',
|
||||||
"ports": [{"containerPort": 8080}]}]}}
|
"ports": [{"containerPort": 8080}]}]}}
|
||||||
|
if failing_probe:
|
||||||
|
for container in template["spec"]["containers"]:
|
||||||
|
container["readinessProbe"] = {"httpGet": {"path": "/healthz",
|
||||||
|
"port": 8089},
|
||||||
|
"initialDelaySeconds": 2,
|
||||||
|
"timeoutSeconds": 1}
|
||||||
spec = k8s_client.V1DeploymentSpec(
|
spec = k8s_client.V1DeploymentSpec(
|
||||||
replicas=3,
|
replicas=3,
|
||||||
selector={"matchLabels": {"app": "demo"}}, template=template)
|
selector={"matchLabels": {"app": "demo"}}, template=template)
|
||||||
@ -440,7 +454,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||||||
# NOTE(juriarte): Wait timeout increased from 180 to 300 in order to
|
# NOTE(juriarte): Wait timeout increased from 180 to 300 in order to
|
||||||
# give the pods time to transition to ready status in the gates (and
|
# give the pods time to transition to ready status in the gates (and
|
||||||
# slow environments).
|
# slow environments).
|
||||||
self.wait_for_status(300, 15, self.get_pods_ready_num,
|
self.wait_for_status(300, 15, self.check_pods_ready_num,
|
||||||
namespace=namespace, label=label,
|
namespace=namespace, label=label,
|
||||||
num_pods=replicas)
|
num_pods=replicas)
|
||||||
|
|
||||||
@ -1622,3 +1636,14 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||||||
raise lib_exc.TimeoutException("Expected num of members is %s but"
|
raise lib_exc.TimeoutException("Expected num of members is %s but"
|
||||||
" actual is %s" % (expected_members,
|
" actual is %s" % (expected_members,
|
||||||
num_members))
|
num_members))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_pod_containers_restarts(cls, pod_names, namespace='default'):
|
||||||
|
containers = {}
|
||||||
|
for pod_name in pod_names:
|
||||||
|
containers[pod_name] = {}
|
||||||
|
pod = cls.k8s_client.CoreV1Api().read_namespaced_pod(pod_name,
|
||||||
|
namespace)
|
||||||
|
for container in pod.status.container_statuses:
|
||||||
|
containers[pod_name][container.name] = container.restart_count
|
||||||
|
return containers
|
||||||
|
@ -19,5 +19,7 @@ POD_AFFINITY = {'requiredDuringSchedulingIgnoredDuringExecution': [
|
|||||||
'topologyKey': 'kubernetes.io/hostname'}]}
|
'topologyKey': 'kubernetes.io/hostname'}]}
|
||||||
TIME_TO_APPLY_SGS = 30
|
TIME_TO_APPLY_SGS = 30
|
||||||
POD_STATUS_RETRIES = 240
|
POD_STATUS_RETRIES = 240
|
||||||
|
POD_CHECK_TIMEOUT = 240
|
||||||
|
POD_CHECK_SLEEP_TIME = 5
|
||||||
NS_TIMEOUT = 600
|
NS_TIMEOUT = 600
|
||||||
REPETITIONS_PER_BACKEND = 10
|
REPETITIONS_PER_BACKEND = 10
|
||||||
|
@ -18,10 +18,12 @@ import kubernetes
|
|||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
from tempest import config
|
from tempest import config
|
||||||
from tempest.lib.common.utils import data_utils
|
from tempest.lib.common.utils import data_utils
|
||||||
|
from tempest.lib.common.utils import test_utils
|
||||||
from tempest.lib import decorators
|
from tempest.lib import decorators
|
||||||
from tempest.lib import exceptions as lib_exc
|
from tempest.lib import exceptions as lib_exc
|
||||||
|
|
||||||
from kuryr_tempest_plugin.tests.scenario import base
|
from kuryr_tempest_plugin.tests.scenario import base
|
||||||
|
from kuryr_tempest_plugin.tests.scenario import consts
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
CONF = config.CONF
|
CONF = config.CONF
|
||||||
@ -340,3 +342,59 @@ class TestLoadBalancerReconciliationScenario(base.BaseKuryrScenarioTest):
|
|||||||
# if there is a connectivity now, that means the LoadBalancer
|
# if there is a connectivity now, that means the LoadBalancer
|
||||||
# is reconciled
|
# is reconciled
|
||||||
self.check_service_internal_connectivity(service_name=service_name)
|
self.check_service_internal_connectivity(service_name=service_name)
|
||||||
|
|
||||||
|
|
||||||
|
class TestServiceWithNotReadyEndpoints(base.BaseKuryrScenarioTest):
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def skip_checks(cls):
|
||||||
|
super(TestServiceWithNotReadyEndpoints, cls).skip_checks()
|
||||||
|
if not CONF.kuryr_kubernetes.service_tests_enabled:
|
||||||
|
raise cls.skipException("Service tests are not enabled")
|
||||||
|
if not CONF.kuryr_kubernetes.containerized:
|
||||||
|
raise cls.skipException("Only runs on containerized setups")
|
||||||
|
|
||||||
|
@decorators.idempotent_id('bddf5441-1244-450d-a125-b5fdcfa1a7b0')
|
||||||
|
def test_service_with_not_ready_endpoints(self):
|
||||||
|
# Create a deployment with a failing probe
|
||||||
|
deployment_name, _ = self.create_deployment(failing_probe=True)
|
||||||
|
|
||||||
|
# Wait until the deployment's pods are running
|
||||||
|
res = test_utils.call_until_true(
|
||||||
|
self.check_pods_status_num, consts.POD_CHECK_TIMEOUT*3,
|
||||||
|
consts.POD_CHECK_SLEEP_TIME, namespace='default', label='app=demo',
|
||||||
|
num_pods=3)
|
||||||
|
self.assertTrue(res, 'Timed out waiting for pods to be running')
|
||||||
|
|
||||||
|
# Wait until the pods are not ready
|
||||||
|
self.wait_for_status(
|
||||||
|
consts.POD_CHECK_TIMEOUT*3, consts.POD_CHECK_SLEEP_TIME,
|
||||||
|
self.check_pods_ready_num, namespace='default', label='app=demo',
|
||||||
|
num_pods=0)
|
||||||
|
|
||||||
|
# Get current Kuryr pods restart count (for a later comparison)
|
||||||
|
controller_pods = self.get_controller_pod_names()
|
||||||
|
container_restarts_before = self.get_pod_containers_restarts(
|
||||||
|
pod_names=controller_pods,
|
||||||
|
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
|
||||||
|
|
||||||
|
# Create a service
|
||||||
|
service_name, _ = self.create_service(pod_label={"app": "demo"},
|
||||||
|
spec_type='ClusterIP')
|
||||||
|
self.addCleanup(self.delete_service, service_name)
|
||||||
|
|
||||||
|
# Check Kuryr pods are not restarted
|
||||||
|
self.check_controller_pod_status_for_time_period(
|
||||||
|
retry_attempts=10,
|
||||||
|
time_between_attempts=3)
|
||||||
|
|
||||||
|
# Get current Kuryr pods restart count
|
||||||
|
container_restarts_after = self.get_pod_containers_restarts(
|
||||||
|
pod_names=controller_pods,
|
||||||
|
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
|
||||||
|
|
||||||
|
# Compare Kuryr pods restart count with previously stored data
|
||||||
|
self.assertEqual(container_restarts_before, container_restarts_after,
|
||||||
|
"Kuryr controller pod(s) were restarted during the "
|
||||||
|
"service creation, expected: %s, obtained: %s" %
|
||||||
|
(container_restarts_before, container_restarts_after))
|
||||||
|
Loading…
Reference in New Issue
Block a user