OCP-Router - add support for OCP routes

This patch extends Kuryr's tempest plugin to verify
OCP-Route capability.

This patch includes also a generic function (named 'wait_kuryr_annotation')
for waiting (with max timeout) till specific annotation appears in object metadata.
A follow-up patch will update the current code to rely on 'wait_kuryr_annotation'
function.

Depends-On: Ia1816d2fa1468385544bb854af67c391050f3636
Change-Id: I8449ceb0735a9aea2e52d5d71a91a51dfd512c17

.zuul.yaml

@@ -45,3 +45,4 @@
         - kuryr-kubernetes-tempest-daemon-containerized-octavia-namespace
         - kuryr-kubernetes-tempest-daemon-containerized-openshift-octavia-serial
         - kuryr-kubernetes-tempest-daemon-ovn
+        - kuryr-kubernetes-tempest-daemon-openshift-octavia-ingress

kuryr_tempest_plugin/config.py

@@ -64,7 +64,12 @@ kubernetes_project_name = cfg.StrOpt("kubernetes_project_name",
                                      default="k8s",
                                      help="The OpenStack project name "
                                           "for Kubernetes")
+
 npwg_multi_vif_enabled = cfg.BoolOpt("npwg_multi_vif_enabled",
                                      default=False,
                                      help="Whether or not NPWG multi-vif "
                                           "feature is enabled")
+
+ocp_router_fip = cfg.StrOpt("ocp_router_fip",
+                            default=None,
+                            help="OCP Router floating IP")

kuryr_tempest_plugin/plugin.py

@@ -52,6 +52,8 @@ class KuryrTempestPlugin(plugins.TempestPlugin):
                           group='kuryr_kubernetes')
         conf.register_opt(project_config.npwg_multi_vif_enabled,
                           group='kuryr_kubernetes')
+        conf.register_opt(project_config.ocp_router_fip,
+                          group='kuryr_kubernetes')
 
     def get_opt_lists(self):
         return [('service_available', [project_config.service_option]),
@@ -61,6 +63,8 @@ class KuryrTempestPlugin(plugins.TempestPlugin):
                                       project_config.containerized,
                                       project_config.kube_system_namespace,
                                       project_config.run_tests_serial,
-                                      project_config.npwg_multi_vif_enabled]),
+                                      project_config.npwg_multi_vif_enabled,
+                                      project_config.ocp_router_fip]),
+
                 ('vif_pool', [project_config.ports_pool_batch,
                               project_config.lb_build_timeout])]

kuryr_tempest_plugin/tests/scenario/base.py

@@ -23,6 +23,7 @@ import kubernetes
 from kubernetes import client as k8s_client
 from kubernetes import config as k8s_config
 from kubernetes.stream import stream
+from openshift import dynamic
 
 from tempest import config
 from tempest.lib.common.utils import data_utils
@@ -37,6 +38,8 @@ KURYR_NET_CRD_VERSION = 'v1'
 KURYR_NET_CRD_PLURAL = 'kuryrnets'
 K8S_ANNOTATION_PREFIX = 'openstack.org/kuryr'
 K8S_ANNOTATION_LBAAS_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-state'
+K8S_ANNOTATION_LBAAS_RT_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-route-state'
+KURYR_ROUTE_DEL_VERIFY_TIMEOUT = 30
 
 
 class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
@@ -58,6 +61,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
         cls.pod_fips = []
         # TODO(dmellado): Config k8s client in a cleaner way
         k8s_config.load_kube_config()
+        cls.dyn_client = dynamic.DynamicClient(
+            k8s_config.new_client_from_config())
 
     @classmethod
     def resource_cleanup(cls):
@@ -285,6 +290,24 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
             else:
                 raise lib_exc.NotImplemented()
 
+    @classmethod
+    def wait_kuryr_annotation(cls, api_version, kind, annotation,
+                              timeout_period, name, namespace='default'):
+        dyn_resource = cls.dyn_client.resources.get(
+            api_version=api_version, kind=kind)
+        start = time.time()
+        while time.time() - start < timeout_period:
+            time.sleep(1)
+            resource = dyn_resource.get(name, namespace=namespace)
+            if resource.metadata.annotations is None:
+                continue
+            for resp_annotation in resource.metadata.annotations:
+                if annotation in resp_annotation:
+                    return
+            LOG.info("Waiting till %s will appear "
+                     "in %s/%s annotation ", annotation, kind, name)
+        raise lib_exc.ServerFault()
+
     @classmethod
     def wait_service_status(cls, service_ip, timeout_period):
         session = requests.Session()
@@ -305,7 +328,9 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
     @classmethod
     def create_setup_for_service_test(cls, pod_num=2, spec_type="ClusterIP",
                                       protocol="TCP", label=None,
-                                      namespace="default", get_ip=True):
+                                      namespace="default", get_ip=True,
+                                      service_name=None):
+
         label = label or data_utils.rand_name('kuryr-app')
         for i in range(pod_num):
             pod_name, pod = cls.create_pod(
@@ -315,11 +340,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
         cls.pod_num = pod_num
         service_name, service_obj = cls.create_service(
             pod_label=pod.metadata.labels, spec_type=spec_type,
-            protocol=protocol, namespace=namespace)
+            protocol=protocol, namespace=namespace, service_name=service_name)
         if get_ip:
             cls.service_ip = cls.get_service_ip(
                 service_name, spec_type=spec_type, namespace=namespace)
             cls.verify_lbaas_endpoints_configured(service_name)
+            cls.service_name = service_name
             cls.wait_service_status(
                 cls.service_ip, CONF.kuryr_kubernetes.lb_build_timeout)
 
@@ -437,6 +463,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
     @classmethod
     def _verify_endpoints_annotation(cls, ep_name, ann_string,
                                      poll_interval=1, namespace='default'):
+        LOG.info("Look for %s string in ep=%s annotation ",
+                 ann_string, ep_name)
         # wait until endpoint annotation created
         while True:
             time.sleep(poll_interval)
@@ -445,6 +473,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
             annotations = ep.metadata.annotations
             try:
                 json.loads(annotations[ann_string])
+                LOG.info("Found %s string in ep=%s annotation ",
+                         ann_string, ep_name)
                 return
             except KeyError:
                 LOG.info("Waiting till %s will appears "
@@ -487,3 +517,90 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
                     'Kuryr controller is not in the %s state' % status
                 )
             retry_attempts -= 1
+
+    @classmethod
+    def create_route(cls, name, hostname, target_svc, namespace='default'):
+        route_manifest = {
+            'apiVersion': 'v1',
+            'kind': 'Route',
+            'metadata':
+                {
+                    'name': name,
+                },
+            'spec':
+                {
+                    'host': hostname,
+                    'to':
+                        {
+                            'kind': 'Service',
+                            'name': target_svc
+                        }
+                }
+        }
+
+        v1_routes = cls.dyn_client.resources.get(api_version='v1',
+                                                 kind='Route')
+        v1_routes.create(body=route_manifest, namespace=namespace)
+
+        LOG.info("Route=%s created, wait for kuryr-annotation", name)
+        cls.wait_kuryr_annotation(
+            api_version='route.openshift.io/v1', kind='Route',
+            annotation='openstack.org/kuryr-route-state',
+            timeout_period=90, name=name)
+        LOG.info("Found %s string in Route=%s annotation ",
+                 'openstack.org/kuryr-route-state', name)
+
+    @classmethod
+    def verify_route_endpoints_configured(cls, ep_name, namespace='default'):
+        cls._verify_endpoints_annotation(
+            ep_name=ep_name, ann_string=K8S_ANNOTATION_LBAAS_RT_STATE,
+            poll_interval=3)
+
+    @classmethod
+    def delete_route(cls, name, namespace='default'):
+        v1_routes = cls.dyn_client.resources.get(api_version='v1',
+                                                 kind='Route')
+        try:
+            v1_routes.delete(name, namespace=namespace)
+        except Exception as e:
+            if e.status == 404:
+                return
+            raise
+
+        # FIXME(yboaron): Use other method (instead of constant delay)
+        # to verify that route was deleted
+        time.sleep(KURYR_ROUTE_DEL_VERIFY_TIMEOUT)
+
+    def verify_route_http(self, router_fip, hostname, amount,
+                          should_succeed=True):
+        LOG.info("Trying to curl the route, Router_FIP=%s, hostname=%s, "
+                 "should_succeed=%s", router_fip, hostname, should_succeed)
+
+        def req():
+            if should_succeed:
+                # FIXME(yboaron): From some reason for route use case,
+                # sometimes only one of service's pods is responding to CURL
+                # although all members and L7 policy were created at Octavia.
+                # so as workaround - I added a constant delay
+                time.sleep(1)
+
+            resp = requests.get('http://{}'.format(router_fip),
+                                headers={'Host': hostname})
+            return resp.status_code, resp.content
+
+        def pred(tester, responses):
+            contents = []
+            for resp in responses:
+                status_code, content = resp
+                if should_succeed:
+                    contents.append(content)
+                else:
+                    tester.assertEqual(requests.codes.SERVICE_UNAVAILABLE,
+                                       status_code)
+            if should_succeed:
+                unique_resps = set(contents)
+                tester.assertEqual(amount, len(unique_resps),
+                                   'Incorrect amount of unique backends. '
+                                   'Got {}'.format(unique_resps))
+
+        self._run_threaded_and_assert(req, pred, fn_timeout=10)

kuryr_tempest_plugin/tests/scenario/test_ocp_route.py

@@ -0,0 +1,92 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from kuryr_tempest_plugin.tests.scenario import base
+from oslo_log import log as logging
+from tempest import config
+from tempest.lib.common.utils import data_utils
+from tempest.lib import decorators
+
+
+LOG = logging.getLogger(__name__)
+CONF = config.CONF
+
+FIRST_ROUTE_NAME = 'firstroute'
+FIRST_ROUTE_HOST_NAME = 'www.first.com'
+SECOND_ROUTE_NAME = 'secondroute'
+SECOND_ROUTE_HOST_NAME = 'www.second.com'
+
+
+class TestOcpRouteScenario(base.BaseKuryrScenarioTest):
+
+    @classmethod
+    def skip_checks(cls):
+        super(TestOcpRouteScenario, cls).skip_checks()
+        if CONF.kuryr_kubernetes.ocp_router_fip is None:
+            raise cls.skipException(
+                "OCP router fip should be specified to run this tests.")
+
+    @decorators.idempotent_id('bddf0001-1244-449d-a125-b5fdcfa1a7a9')
+    def test_create_route_after_service(self):
+        self.create_setup_for_service_test()
+        self.addCleanup(self.delete_route, FIRST_ROUTE_NAME)
+        self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME,
+                          self.service_name)
+
+        self.verify_route_endpoints_configured(self.service_name)
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               FIRST_ROUTE_HOST_NAME,
+                               self.pod_num)
+        self.delete_route(FIRST_ROUTE_NAME)
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               FIRST_ROUTE_HOST_NAME,
+                               self.pod_num,
+                               should_succeed=False)
+
+    @decorators.idempotent_id('bddf0002-1244-449d-a125-b5fdcfa1a7a9')
+    def test_two_routes_same_service(self):
+        self.create_setup_for_service_test()
+        self.addCleanup(self.delete_route, FIRST_ROUTE_NAME)
+        self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME,
+                          self.service_name)
+        self.addCleanup(self.delete_route, SECOND_ROUTE_NAME)
+        self.create_route(SECOND_ROUTE_NAME, SECOND_ROUTE_HOST_NAME,
+                          self.service_name)
+        self.verify_route_endpoints_configured(self.service_name)
+
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               FIRST_ROUTE_HOST_NAME,
+                               self.pod_num)
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               SECOND_ROUTE_HOST_NAME,
+                               self.pod_num)
+        self.delete_route(FIRST_ROUTE_NAME)
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               SECOND_ROUTE_HOST_NAME,
+                               self.pod_num)
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               FIRST_ROUTE_HOST_NAME,
+                               self.pod_num,
+                               should_succeed=False)
+
+    @decorators.idempotent_id('bddf0003-1244-449d-a125-b5fdcfa1a7a9')
+    def test_create_route_before_service(self):
+        service_name = data_utils.rand_name(prefix='kuryr-service')
+        self.addCleanup(self.delete_route, FIRST_ROUTE_NAME)
+        self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME,
+                          service_name)
+        self.create_setup_for_service_test(service_name=service_name)
+        self.verify_route_endpoints_configured(service_name)
+        self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
+                               FIRST_ROUTE_HOST_NAME,
+                               self.pod_num)

requirements.txt

@@ -11,3 +11,4 @@ tempest>=17.1.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 testtools>=2.2.0 # MIT
 kubernetes>=5.0.0 # Apache-2.0
+openshift>=0.7.0