openstack/kuryr-tempest-plugin
Code Issues Proposed changes

Limit number of K8s API requests

Browse Source 
As the K8s client is logging every response to every request, we're
starting to hit limits of tox' or Zuul's log side limit. This commit
attempts to limit number of requests by making sure that in case of
status checks we're iterating a list of pods instead of calling API for
every pod. Moreover many `time.sleep()` occurences are modified to
reduce the number of the requests made.

Change-Id: Ifc2dfce2405429bbcae8c01f13f06d4e9fae9c8a
This commit is contained in:
Michał Dulko 2022-02-22 12:39:46 +01:00
parent 85807e55a9
commit 86423cc26c
5 changed files with 39 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
kuryr_tempest_plugin/tests/scenario/base.py
View File

@ -228,11 +228,11 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
@classmethod
def wait_for_pod_status(cls, pod_name, namespace="default",
pod_status=None, retries=30):
pod_status=None, retries=6):
while pod_status != cls.get_pod_status(
pod_name,
namespace=namespace):
time.sleep(1)
time.sleep(5)
retries -= 1
if retries == 0:
raise lib_exc.TimeoutException()
@ -273,31 +273,33 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
except kubernetes.client.rest.ApiException:
return False
return cls.get_readiness_state_from_pod(pod,
container_name=container_name)
@classmethod
def get_readiness_state_from_pod(cls, pod, container_name=None):
if container_name:
for container in pod.status.container_statuses:
if container.name == container_name:
return container.ready
else:
elif pod.status.conditions:
for condition in pod.status.conditions:
if condition.type == 'Ready':
return condition.status == 'True'
return False
@classmethod
def check_pods_ready_num(cls, namespace="default",
label="", num_pods=1):
pods = cls.get_pod_name_list(namespace=namespace,
label_selector=label)
ready_pods = sum([cls.get_readiness_state(p) for p in pods])
return (num_pods == ready_pods)
def check_pods_ready_num(cls, namespace="default", label="", num_pods=1):
pods = cls.get_pod_list(namespace=namespace, label_selector=label)
ready_pods = sum([cls.get_readiness_state_from_pod(p) for p in pods])
return num_pods == ready_pods
@classmethod
def check_pods_status_num(cls, namespace="default", label="", num_pods=1,
status="Running"):
pods = cls.get_pod_name_list(namespace=namespace,
label_selector=label)
status_pods = sum([cls.get_pod_status(p) == status for p in pods])
return (num_pods == status_pods)
pods = cls.get_pod_list(namespace=namespace, label_selector=label)
status_pods = sum([p.status.phase == status for p in pods])
return num_pods == status_pods
@classmethod
def get_pod_readiness(cls, pod_name, namespace="default"):
@ -921,7 +923,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
kuryr_crd_annotation = K8S_ANNOTATION_PREFIX + "-net-crd"
# wait until namespace gets created
while True:
time.sleep(1)
time.sleep(10)
ns = cls.k8s_client.CoreV1Api().read_namespace_status(name)
if (ns.metadata.annotations and
ns.metadata.annotations.get(kuryr_crd_annotation)):

30
kuryr_tempest_plugin/tests/scenario/base_network_policy.py
View File

@ -50,7 +50,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
ready = False
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, ready = self.get_np_crd_info(
name=network_policy_name, namespace=namespace)
if sg_id and ready:
@ -95,7 +95,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
egress_cidrs_found))):
rules_match = True
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if not rules_match:
msg = 'Timed out waiting sg rules for np %s to match' % np
@ -124,7 +124,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, ready = self.get_np_crd_info(
network_policy_name, namespace=namespace_name)
if sg_id and ready:
@ -206,7 +206,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name,
namespace=namespace_name)
if sg_id:
@ -247,7 +247,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
self.get_np_crd_info(network_policy_name,
namespace=namespace_name)
except kubernetes.client.rest.ApiException as e:
@ -277,7 +277,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name)
if sg_id:
break
@ -291,7 +291,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
self.delete_network_policy(network_policy_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if network_policy_name in self.list_network_policies():
continue
try:
@ -320,7 +320,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
_, crd_pod_selector, _ = self.get_np_crd_info(
network_policy_name)
if crd_pod_selector:
@ -341,7 +341,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
_, crd_pod_selector, _ = self.get_np_crd_info(
network_policy_name)
labels = crd_pod_selector.get('matchLabels')
@ -368,7 +368,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name,
namespace=ns_name)
if sg_id:
@ -384,7 +384,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
self.delete_namespace(ns_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if network_policy_name in self.list_network_policies(
namespace=ns_name):
continue
@ -398,7 +398,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
sg_ids_after = [sg['id'] for sg in sgs_after]
if sg_id not in sg_ids_after:
break
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if time.time() - start >= TIMEOUT_PERIOD:
raise lib_exc.TimeoutException('Sec group ID still exists')
@ -416,7 +416,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name,
namespace=ns_name)
if sg_id:
@ -430,7 +430,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
self.delete_network_policy(network_policy_name, namespace=ns_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if network_policy_name in self.list_network_policies(
namespace=ns_name):
continue
@ -444,7 +444,7 @@ class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
sg_ids_after = [sg['id'] for sg in sgs_after]
if sg_id not in sg_ids_after:
break
time.sleep(1)
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if time.time() - start >= TIMEOUT_PERIOD:
raise lib_exc.TimeoutException('Sec group ID still exists')

1
kuryr_tempest_plugin/tests/scenario/consts.py
View File

@ -21,5 +21,6 @@ TIME_TO_APPLY_SGS = 30
POD_STATUS_RETRIES = 240
POD_CHECK_TIMEOUT = 240
POD_CHECK_SLEEP_TIME = 5
NP_CHECK_SLEEP_TIME = 10
NS_TIMEOUT = 600
REPETITIONS_PER_BACKEND = 10

4
kuryr_tempest_plugin/tests/scenario/test_kuryr_restart.py
View File

@ -51,11 +51,11 @@ class TestKuryrRestartScenario(base.BaseKuryrScenarioTest):
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
# make sure the kuryr pod was deleted
pod_delete_retries = 30
pod_delete_retries = 6
while self.get_pod_status(
kuryr_pod_name,
namespace=CONF.kuryr_kubernetes.kube_system_namespace):
time.sleep(1)
time.sleep(5)
pod_delete_retries -= 1
if pod_delete_retries == 0:
raise lib_exc.TimeoutException()

10
kuryr_tempest_plugin/tests/scenario/test_namespace.py
View File

@ -275,7 +275,7 @@ class TestNamespaceScenario(base.BaseKuryrScenarioTest):
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(1)
time.sleep(10)
try:
if CONF.kuryr_kubernetes.kuryrnetworks:
self.get_kuryr_network_crds(namespace)
@ -286,7 +286,7 @@ class TestNamespaceScenario(base.BaseKuryrScenarioTest):
# Also wait for the namespace removal
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(1)
time.sleep(10)
try:
self.get_namespace(namespace)
except kubernetes.client.rest.ApiException:
@ -390,11 +390,11 @@ class TestNamespaceScenario(base.BaseKuryrScenarioTest):
self.delete_namespace(ns_name)
# wait for namespace to be deleted
# FIXME(itzikb) Set retries to 120 when BZ#1997120 is fixed
retries = 600
# FIXME(itzikb) Set retries to 24 when BZ#1997120 is fixed
retries = 120
while True:
try:
time.sleep(1)
time.sleep(5)
self.k8s_client.CoreV1Api().read_namespace(ns_name)
retries -= 1
self.assertNotEqual(0, retries,