Michał Dulko 86423cc26c Limit number of K8s API requests
As the K8s client is logging every response to every request, we're
starting to hit limits of tox' or Zuul's log side limit. This commit
attempts to limit number of requests by making sure that in case of
status checks we're iterating a list of pods instead of calling API for
every pod. Moreover many `time.sleep()` occurences are modified to
reduce the number of the requests made.

Change-Id: Ifc2dfce2405429bbcae8c01f13f06d4e9fae9c8a
2022-03-01 13:17:52 +01:00

635 lines
28 KiB
Python

# Copyright 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import time
import kubernetes
from kubernetes import client as k8s_client
import netaddr
from oslo_log import log as logging
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
from tempest.lib import exceptions as lib_exc
from kuryr_tempest_plugin.tests.scenario import base
from kuryr_tempest_plugin.tests.scenario import consts
LOG = logging.getLogger(__name__)
CONF = config.CONF
TIMEOUT_PERIOD = 180
class TestNetworkPolicyScenario(base.BaseKuryrScenarioTest,
metaclass=abc.ABCMeta):
@classmethod
def skip_checks(cls):
super(TestNetworkPolicyScenario, cls).skip_checks()
if not CONF.kuryr_kubernetes.network_policy_enabled:
raise cls.skipException('Network Policy driver and handler must '
'be enabled to run this tests')
def get_sg_rules_for_np(self, namespace, network_policy_name):
start = time.time()
sg_id = None
ready = False
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, ready = self.get_np_crd_info(
name=network_policy_name, namespace=namespace)
if sg_id and ready:
break
except kubernetes.client.rest.ApiException:
continue
self.assertIsNotNone(sg_id)
self.assertTrue(ready)
return self.list_security_group_rules(sg_id)
def check_sg_rules_for_np(self, namespace, np,
ingress_cidrs_should_exist=(),
egress_cidrs_should_exist=(),
ingress_cidrs_shouldnt_exist=(),
egress_cidrs_shouldnt_exist=()):
ingress_cidrs_should_exist = set(ingress_cidrs_should_exist)
egress_cidrs_should_exist = set(egress_cidrs_should_exist)
ingress_cidrs_shouldnt_exist = set(ingress_cidrs_shouldnt_exist)
egress_cidrs_shouldnt_exist = set(egress_cidrs_shouldnt_exist)
rules_match = False
start = time.time()
while not rules_match and (time.time() - start) < TIMEOUT_PERIOD:
ingress_cidrs_found = set()
egress_cidrs_found = set()
sg_rules = self.get_sg_rules_for_np(namespace, np)
for rule in sg_rules:
if rule['direction'] == 'ingress':
ingress_cidrs_found.add(rule['remote_ip_prefix'])
elif rule['direction'] == 'egress':
egress_cidrs_found.add(rule['remote_ip_prefix'])
if (ingress_cidrs_should_exist.issubset(ingress_cidrs_found)
and (not ingress_cidrs_shouldnt_exist
or not ingress_cidrs_shouldnt_exist.issubset(
ingress_cidrs_found))
and egress_cidrs_should_exist.issubset(egress_cidrs_found)
and (not egress_cidrs_shouldnt_exist
or not egress_cidrs_shouldnt_exist.issubset(
egress_cidrs_found))):
rules_match = True
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if not rules_match:
msg = 'Timed out waiting sg rules for np %s to match' % np
raise lib_exc.TimeoutException(msg)
@abc.abstractmethod
def get_np_crd_info(self, name, namespace='default', **kwargs):
pass
@decorators.idempotent_id('a9db5bc5-e921-4719-8201-5431537c86f8')
def test_ipblock_network_policy_sg_rules(self):
ingress_ipblock = "5.5.5.0/24"
egress_ipblock = "4.4.4.0/24"
namespace_name, namespace = self.create_namespace()
self.addCleanup(self.delete_namespace, namespace_name)
np = self.create_network_policy(namespace=namespace_name,
ingress_ipblock_cidr=ingress_ipblock,
egress_ipblock_cidr=egress_ipblock,
ingress_port=2500)
LOG.debug("Creating network policy %s", np)
self.addCleanup(self.delete_network_policy, np.metadata.name,
namespace_name)
network_policy_name = np.metadata.name
sg_id = None
ready = False
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, ready = self.get_np_crd_info(
network_policy_name, namespace=namespace_name)
if sg_id and ready:
break
except kubernetes.client.rest.ApiException:
continue
self.assertIsNotNone(sg_id)
self.assertTrue(ready)
sec_group_rules = self.list_security_group_rules(sg_id)
ingress_block_found, egress_block_found = False, False
for rule in sec_group_rules:
if (rule['direction'] == 'ingress' and
rule['remote_ip_prefix'] == ingress_ipblock):
ingress_block_found = True
if (rule['direction'] == 'egress' and
rule['remote_ip_prefix'] == egress_ipblock):
egress_block_found = True
self.assertTrue(ingress_block_found)
self.assertTrue(egress_block_found)
@decorators.idempotent_id('a9db5bc5-e921-4819-8301-5431437c76f8')
def test_ipblock_network_policy_allow_except(self):
namespace_name, namespace = self.create_namespace()
self.addCleanup(self.delete_namespace, namespace_name)
pod_name, pod = self.create_pod(namespace=namespace_name)
self.addCleanup(self.delete_pod, pod_name, pod,
namespace=namespace_name)
if CONF.kuryr_kubernetes.kuryrnetworks:
cidr = self.get_kuryr_network_crds(
namespace_name)['status']['subnetCIDR']
else:
crd_name = 'ns-' + namespace_name
cidr = self.get_kuryr_net_crds(
crd_name)['spec']['subnetCIDR']
ipn = netaddr.IPNetwork(cidr)
max_prefixlen = "/32"
if ipn.version == 6:
max_prefixlen = "/128"
curl_tmpl = self.get_curl_template(cidr, extra_args='-m 5', port=True)
allow_all_cidr = cidr
pod_ip_list = []
pod_name_list = []
cmd_list = []
for i in range(4):
pod_name, pod = self.create_pod(namespace=namespace_name)
self.addCleanup(self.delete_pod, pod_name, pod,
namespace=namespace_name)
pod_name_list.append(pod_name)
pod_ip = self.get_pod_ip(pod_name, namespace=namespace_name)
pod_ip_list.append(pod_ip)
cmd = ["/bin/sh", "-c", curl_tmpl.format(pod_ip_list[i], ':8080')]
cmd_list.append(cmd)
# Check connectivity from pod4 to other pods before creating NP
for i in range(3):
self.assertIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[3], cmd_list[i], namespace=namespace_name))
# Check connectivity from pod1 to pod4 before creating NP
self.assertIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[0], cmd_list[3], namespace=namespace_name))
# Create NP allowing all besides first pod on ingress
# and second pod on egress
np = self.create_network_policy(
namespace=namespace_name, ingress_ipblock_cidr=allow_all_cidr,
ingress_ipblock_except=[pod_ip_list[0] + max_prefixlen],
egress_ipblock_cidr=allow_all_cidr,
egress_ipblock_except=[pod_ip_list[1] + max_prefixlen])
LOG.debug("Creating network policy %s", np)
network_policy_name = np.metadata.name
sg_id = None
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name,
namespace=namespace_name)
if sg_id:
break
except kubernetes.client.rest.ApiException:
continue
if not sg_id:
msg = ('Timed out waiting for knp %s creation' %
network_policy_name)
raise lib_exc.TimeoutException(msg)
# Wait for network policy to be created
time.sleep(consts.TIME_TO_APPLY_SGS)
# Check that http connection from pod1 to pod4 is blocked
# after creating NP
self.assertNotIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[0], cmd_list[3], namespace=namespace_name))
# Check that http connection from pod4 to pod2 is blocked
# after creating NP
self.assertNotIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[3], cmd_list[1], namespace=namespace_name))
# Check that http connection from pod4 to pod1 is not blocked
self.assertIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[3], cmd_list[0], namespace=namespace_name))
# Check that there is still http connection to pod3
# from pod4 as it's not blocked by IPblock rules
self.assertIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[3], cmd_list[2], namespace=namespace_name))
# Delete network policy and check that there is still http connection
# between pods
self.delete_network_policy(np.metadata.name, namespace_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
self.get_np_crd_info(network_policy_name,
namespace=namespace_name)
except kubernetes.client.rest.ApiException as e:
if e.status == 404:
break
else:
continue
else:
msg = ('Timed out waiting for knp %s deletion' %
network_policy_name)
raise lib_exc.TimeoutException(msg)
for i in range(3):
self.assertIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[3], cmd_list[i], namespace=namespace_name))
for i in range(1, 4):
self.assertIn(consts.POD_OUTPUT, self.exec_command_in_pod(
pod_name_list[0], cmd_list[i], namespace=namespace_name))
@decorators.idempotent_id('24577a9b-1d29-409b-8b60-da3b49d776b1')
def test_create_delete_network_policy(self):
np = self.create_network_policy()
LOG.debug("Creating network policy %s" % np)
network_policy_name = np.metadata.name
network_policies = self.list_network_policies()
sg_id = None
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name)
if sg_id:
break
except kubernetes.client.rest.ApiException:
continue
self.assertIsNotNone(sg_id)
sgs = self.list_security_groups(fields='id')
sg_ids = [sg['id'] for sg in sgs]
self.assertIn(network_policy_name, network_policies)
self.assertIn(sg_id, sg_ids)
self.delete_network_policy(network_policy_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if network_policy_name in self.list_network_policies():
continue
try:
self.get_np_crd_info(network_policy_name)
except kubernetes.client.rest.ApiException:
break
sgs_after = self.list_security_groups(fields='id')
sg_ids_after = [sg['id'] for sg in sgs_after]
self.assertNotIn(sg_id, sg_ids_after)
@decorators.idempotent_id('44daf8fe-6a4b-4ca9-8685-97fb1f573e5e')
def test_update_network_policy(self):
"""Update a Network Policy with a new pod selector
This method creates a Network Policy with a specific pod selector
and updates it with a new pod selector. The CRD should always have
the same pod selector as the Policy.
"""
match_labels = {'app': 'demo'}
np = self.create_network_policy(match_labels=match_labels)
self.addCleanup(self.delete_network_policy, np.metadata.name)
network_policy_name = np.metadata.name
crd_pod_selector = None
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
_, crd_pod_selector, _ = self.get_np_crd_info(
network_policy_name)
if crd_pod_selector:
break
except kubernetes.client.rest.ApiException:
continue
self.assertIsNotNone(crd_pod_selector)
self.assertEqual(crd_pod_selector.get('matchLabels'), match_labels)
label_key = 'context'
match_labels = {label_key: 'demo'}
np = self.read_network_policy(np)
np.spec.pod_selector.match_labels = match_labels
np = self.update_network_policy(np)
labels = {}
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
_, crd_pod_selector, _ = self.get_np_crd_info(
network_policy_name)
labels = crd_pod_selector.get('matchLabels')
if labels.get(label_key):
break
except kubernetes.client.rest.ApiException:
continue
if not labels.get(label_key):
raise lib_exc.TimeoutException()
self.assertEqual(labels, match_labels)
@decorators.idempotent_id('24577a9b-1d29-409b-8b60-da3c49d777c2')
def test_delete_namespace_with_network_policy(self):
ns_name, ns = self.create_namespace()
match_labels = {'role': 'db'}
np = self.create_network_policy(match_labels=match_labels,
namespace=ns_name)
LOG.debug("Creating network policy %s" % np)
network_policy_name = np.metadata.name
network_policies = self.list_network_policies(namespace=ns_name)
sg_id = None
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name,
namespace=ns_name)
if sg_id:
break
except kubernetes.client.rest.ApiException:
continue
sgs = self.list_security_groups(fields='id')
sg_ids = [sg['id'] for sg in sgs]
self.assertIn(network_policy_name, network_policies)
self.assertIn(sg_id, sg_ids)
# delete namespace
self.delete_namespace(ns_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if network_policy_name in self.list_network_policies(
namespace=ns_name):
continue
try:
self.get_np_crd_info(network_policy_name, namespace=ns_name)
except kubernetes.client.rest.ApiException:
break
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
sgs_after = self.list_security_groups(fields='id')
sg_ids_after = [sg['id'] for sg in sgs_after]
if sg_id not in sg_ids_after:
break
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if time.time() - start >= TIMEOUT_PERIOD:
raise lib_exc.TimeoutException('Sec group ID still exists')
@decorators.idempotent_id('24577a9b-1d46-419b-8b60-da3c49d777c3')
def test_create_delete_network_policy_non_default_ns(self):
ns_name, ns = self.create_namespace()
self.addCleanup(self.delete_namespace, ns_name)
match_labels = {'role': 'db'}
np = self.create_network_policy(match_labels=match_labels,
namespace=ns_name)
LOG.debug("Creating network policy %s" % np)
network_policy_name = np.metadata.name
network_policies = self.list_network_policies(namespace=ns_name)
sg_id = None
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
try:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
sg_id, _, _ = self.get_np_crd_info(network_policy_name,
namespace=ns_name)
if sg_id:
break
except kubernetes.client.rest.ApiException:
continue
sgs = self.list_security_groups(fields='id')
sg_ids = [sg['id'] for sg in sgs]
self.assertIn(network_policy_name, network_policies)
self.assertIn(sg_id, sg_ids)
self.delete_network_policy(network_policy_name, namespace=ns_name)
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if network_policy_name in self.list_network_policies(
namespace=ns_name):
continue
try:
self.get_np_crd_info(network_policy_name, namespace=ns_name)
except kubernetes.client.rest.ApiException:
break
start = time.time()
while time.time() - start < TIMEOUT_PERIOD:
sgs_after = self.list_security_groups(fields='id')
sg_ids_after = [sg['id'] for sg in sgs_after]
if sg_id not in sg_ids_after:
break
time.sleep(consts.NP_CHECK_SLEEP_TIME)
if time.time() - start >= TIMEOUT_PERIOD:
raise lib_exc.TimeoutException('Sec group ID still exists')
@decorators.idempotent_id('a93b5bc5-e931-4719-8201-54315c5c86f8')
def test_network_policy_add_remove_pod(self):
np_name_server = 'allow-all-server'
np_name_client = 'allow-all-client'
server_label = {'app': 'server'}
client_label = {'app': 'client'}
namespace_name, namespace = self.create_namespace()
self.addCleanup(self.delete_namespace, namespace_name)
self.create_setup_for_service_test(label='server',
namespace=namespace_name,
cleanup=False)
LOG.debug("A service %s and two pods were created in namespace %s",
self.service_name, namespace_name)
service_pods = self.get_pod_list(namespace=namespace_name,
label_selector='app=server')
service_pods_cidrs = [pod.status.pod_ip+'/32' for pod in service_pods]
(first_server_pod_cidr, first_server_pod_name) = (
service_pods[0].status.pod_ip+"/32",
service_pods[0].metadata.name)
client_pod_name = self.check_service_internal_connectivity(
namespace=namespace_name,
labels=client_label,
cleanup=False)
client_pod_ip = self.get_pod_ip(client_pod_name,
namespace=namespace_name)
client_pod_cidr = client_pod_ip + "/32"
LOG.debug("Client pod %s was created", client_pod_name)
LOG.debug("Connection to service %s from %s was successful",
self.service_name, client_pod_name)
# Check connectivity in the same namespace
connect_to_service_cmd = ["/bin/sh", "-c", "curl {dst_ip}".format(
dst_ip=self.service_ip)]
blocked_pod, _ = self.create_pod(namespace=namespace_name)
self.assertIn(consts.POD_OUTPUT,
self.exec_command_in_pod(blocked_pod,
connect_to_service_cmd,
namespace_name))
pods_server_match_expression = {'key': 'app',
'operator': 'In',
'values': ['server']}
pods_client_match_expression = {'key': 'app',
'operator': 'In',
'values': ['client']}
np_server = self.create_network_policy(
name=np_name_server,
namespace=namespace_name,
match_labels=server_label,
ingress_match_expressions=[pods_client_match_expression],
egress_match_expressions=[pods_client_match_expression])
LOG.debug("Network policy %s with match expression %s was created",
np_server, pods_server_match_expression)
self.addCleanup(self.delete_network_policy, np_server.metadata.name,
namespace_name)
np_client = self.create_network_policy(
name=np_name_client,
namespace=namespace_name,
match_labels=client_label,
ingress_match_expressions=[pods_server_match_expression],
egress_match_expressions=[pods_server_match_expression])
LOG.debug("Network policy %s with match expression %s was created",
np_client, pods_client_match_expression)
self.addCleanup(self.delete_network_policy, np_client.metadata.name,
namespace_name)
self.check_sg_rules_for_np(
namespace_name, np_name_server,
ingress_cidrs_should_exist=[client_pod_cidr],
egress_cidrs_should_exist=[client_pod_cidr],
ingress_cidrs_shouldnt_exist=[],
egress_cidrs_shouldnt_exist=[])
self.check_sg_rules_for_np(
namespace_name, np_name_client,
ingress_cidrs_should_exist=service_pods_cidrs,
egress_cidrs_should_exist=service_pods_cidrs,
ingress_cidrs_shouldnt_exist=[],
egress_cidrs_shouldnt_exist=[])
self.check_service_internal_connectivity(namespace=namespace_name,
pod_name=client_pod_name)
LOG.debug("Connection to service %s from %s was successful after "
"network policy was applied",
self.service_name, client_pod_name)
# Check no connectivity from a pod not in the NP
self.assertNotIn(consts.POD_OUTPUT,
self.exec_command_in_pod(blocked_pod,
connect_to_service_cmd,
namespace_name))
self.delete_pod(first_server_pod_name, namespace=namespace_name)
LOG.debug("Deleted pod %s from service %s",
first_server_pod_name, self.service_name)
self.verify_lbaas_endpoints_configured(self.service_name,
1, namespace_name)
self.check_service_internal_connectivity(namespace=namespace_name,
pod_name=client_pod_name,
pod_num=1)
LOG.debug("Connection to service %s with one pod from %s was "
"successful", self.service_name, client_pod_name)
# Check that the deleted pod is removed from SG rules
self.check_sg_rules_for_np(
namespace_name, np_name_client,
ingress_cidrs_shouldnt_exist=[
first_server_pod_cidr],
egress_cidrs_shouldnt_exist=[
first_server_pod_cidr])
pod_name, pod = self.create_pod(labels=server_label,
namespace=namespace_name)
LOG.debug("Pod server %s with label %s was created",
pod_name, server_label)
self.verify_lbaas_endpoints_configured(self.service_name,
2, namespace_name)
service_pods = self.get_pod_list(namespace=namespace_name,
label_selector='app=server')
service_pods_cidrs = [pod.status.pod_ip+'/32' for pod in service_pods]
self.check_sg_rules_for_np(
namespace_name, np_name_server,
ingress_cidrs_should_exist=[client_pod_cidr],
egress_cidrs_should_exist=[client_pod_cidr],
ingress_cidrs_shouldnt_exist=[],
egress_cidrs_shouldnt_exist=[])
self.check_sg_rules_for_np(
namespace_name, np_name_client,
ingress_cidrs_should_exist=service_pods_cidrs,
egress_cidrs_should_exist=service_pods_cidrs)
self.check_service_internal_connectivity(namespace=namespace_name,
pod_name=client_pod_name)
LOG.debug("Connection to service %s from %s was successful",
self.service_name, client_pod_name)
# Check no connectivity from a pod not in the NP
self.assertNotIn(consts.POD_OUTPUT,
self.exec_command_in_pod(blocked_pod,
connect_to_service_cmd,
namespace_name))
@decorators.idempotent_id('ee018bf6-2d5d-4c4e-8c79-793f4772852f')
def test_network_policy_hairpin_traffic(self):
namespace_name, namespace = self.create_namespace()
self.addCleanup(self.delete_namespace, namespace_name)
svc_name, svc_pods = self.create_setup_for_service_test(
namespace=namespace_name, cleanup=False, save=False, pod_num=1)
self.check_service_internal_connectivity(
namespace=namespace_name, pod_num=1, service_name=svc_name,
pod_name=svc_pods[0])
policy_name = data_utils.rand_name(prefix='kuryr-policy')
np = k8s_client.V1NetworkPolicy(
kind='NetworkPolicy',
api_version='networking.k8s.io/v1',
metadata=k8s_client.V1ObjectMeta(
name=policy_name,
namespace=namespace_name),
spec=k8s_client.V1NetworkPolicySpec(
pod_selector=k8s_client.V1LabelSelector(),
policy_types=['Egress', 'Ingress'],
ingress=[
k8s_client.V1NetworkPolicyIngressRule(
_from=[
k8s_client.V1NetworkPolicyPeer(
pod_selector=k8s_client.V1LabelSelector(),
),
],
),
],
egress=[
k8s_client.V1NetworkPolicyEgressRule(
to=[
k8s_client.V1NetworkPolicyPeer(
pod_selector=k8s_client.V1LabelSelector(),
),
],
),
],
),
)
k8s_client.NetworkingV1Api().create_namespaced_network_policy(
namespace=namespace_name, body=np)
# Just to wait for SGs.
self.get_sg_rules_for_np(namespace_name, policy_name)
self.check_service_internal_connectivity(
namespace=namespace_name, pod_num=1, service_name=svc_name,
pod_name=svc_pods[0])