From 159fe3e0ae75f774c9329f9ba2e78de603cbd749 Mon Sep 17 00:00:00 2001
From: Yossi Boaron <yossi.boaron.1234@gmail.com>
Date: Tue, 24 Apr 2018 22:24:38 +0300
Subject: [PATCH] OCP-Router: add support for external Ingress Controller's LB
 creation

This is the first patch of the Ingress Controller capability.

In order for the K8S Ingress and OpenShift Route resources to work,
the cluster must have an Ingress Controller running.
The Kuryr's Ingress Controller implementation will be based on
Octavia L7 load balancing.

This patch adds support for the creation of an external Load Balancer
in Devstack deployment, the follow-up Ingress Controller patches will
configure the L7 rules in that LB to perform the actual L7 routing.

Partially Implements: blueprint openshift-router-support
Change-Id: I9c18bd1d2d0f2127a1a924efe7976a38b6f7cc51
---
 devstack/lib/kuryr_kubernetes        | 63 ++++++++++++++++++++++------
 devstack/local.conf.openshift.sample | 14 +++++++
 devstack/plugin.sh                   | 55 +++++++++++++++++++++++-
 devstack/settings                    |  6 +++
 4 files changed, 123 insertions(+), 15 deletions(-)

diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes
index 7a7b07978..14c7193be 100644
--- a/devstack/lib/kuryr_kubernetes
+++ b/devstack/lib/kuryr_kubernetes
@@ -316,10 +316,23 @@ function create_k8s_router_fake_service {
         --os-region "$REGION_NAME" \
         subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
         -f value -c gateway_ip)
+    create_k8s_fake_service $fake_svc_name $router_ip
+}
+
+# create_k8s_fake_service
+#   Description: Creates an endpoint-less kubernetes service to keep Kubernetes
+#                API server from allocating this IP for another service
+function create_k8s_fake_service {
+    local svc_name
+    local svc_ip
+
+    fake_svc_name="$1"
+    fake_svc_ip="$2"
+
     existing_svc_ip=$(/usr/local/bin/kubectl get svc --namespace kube-system -o jsonpath='{.items[?(@.metadata.name=='"\"${fake_svc_name}\""')].spec.clusterIP}')
 
     if [[ "$existing_svc_ip" == "" ]]; then
-        # Create fake router service so the router clusterIP can't be reassigned
+        # Create fake service so the clusterIP can't be reassigned
         cat <<EOF | /usr/local/bin/kubectl create -f -
 kind: Service
 apiVersion: v1
@@ -328,7 +341,7 @@ metadata:
   namespace: kube-system
 spec:
   type: ClusterIP
-  clusterIP: "${router_ip}"
+  clusterIP: "${fake_svc_ip}"
   ports:
     - protocol: TCP
       port: 80
@@ -852,25 +865,30 @@ function wait_for_lb {
 #                or Octavia
 #   Params:
 #       lb_name: Name to give to the load balancer.
-#       lb_vip: Virtual IP to give to the load balancer.
 #       lb_vip_subnet: Id or name of the subnet where lb_vip should be
 #                      allocated.
+#       lb_vip: Virtual IP to give to the load balancer - optional.
 function create_load_balancer {
     local lb_name
-    local lb_vip
     local lb_vip_subnet
+    local lb_params
 
     lb_name="$1"
-    lb_vip="$2"
-    lb_vip_subnet="$3"
-    if is_service_enabled octavia; then
-        openstack loadbalancer create --name "$lb_name" \
-            --vip-address "$lb_vip" \
-            --vip-subnet-id "$lb_vip_subnet"
+    lb_vip_subnet="$2"
+
+    lb_params=" --name $lb_name "
+    if [ -z "$3" ]; then
+        echo -n "create_load_balancer LB=$lb_name, lb_vip not provided."
     else
-        neutron lbaas-loadbalancer-create --name "$lb_name" \
-            --vip-address "$lb_vip" \
-            "$lb_vip_subnet"
+        lb_params+=" --vip-address $3"
+    fi
+
+    if is_service_enabled octavia; then
+        lb_params+=" --vip-subnet-id $lb_vip_subnet"
+        openstack loadbalancer create $lb_params
+    else
+        lb_params+=" $lb_vip_subnet"
+        neutron lbaas-loadbalancer-create $lb_params
     fi
 }
 
@@ -1021,3 +1039,22 @@ first, last = n.subnets(prefixlen_diff=1)
 print("%s\\t%s" % (first, last))
 EOF
 }
+
+# get_loadbalancer_attribute
+#   Description: Get load balancer attribute
+#   Params:
+#       lb_name: Load balancer name
+#       lb_attr: attribute name
+function get_loadbalancer_attribute {
+    local lb_name
+    local lb_attr
+
+    lb_name="$1"
+    lb_attr="$2"
+
+    if is_service_enabled octavia; then
+        openstack loadbalancer show "$lb_name" -c "$lb_attr" -f value
+    else
+        neutron lbaas-loadbalancer-show "$lb_name" -c "$lb_attr" -f value
+    fi
+}
diff --git a/devstack/local.conf.openshift.sample b/devstack/local.conf.openshift.sample
index 1d95f4e66..9e00771de 100644
--- a/devstack/local.conf.openshift.sample
+++ b/devstack/local.conf.openshift.sample
@@ -160,6 +160,20 @@ enable_service kuryr-kubernetes
 # Since Rocky release this is a default deployment configuration.
 enable_service kuryr-daemon
 
+# Kuryr enable L7 routing
+# ========================
+#
+# Uncomment the next line to enable the L7 Routing/Ingress controller
+#
+#KURYR_ENABLE_INGRESS=True
+
+# Kuryr L7 router/lb name
+# ========================
+#
+# Edit the next line to change L7 Router/LB name
+#
+#KURYR_L7_ROUTER_NAME=kuryr-l7-router
+
 # Containerized Kuryr
 # ===================
 #
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 22c5d6a2b..2a01e70a0 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -232,8 +232,8 @@ function create_k8s_api_service {
 
     k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
 
-    create_load_balancer "$lb_name" "$k8s_api_clusterip" \
-        "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"
+    create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
+            "$k8s_api_clusterip"
     create_load_balancer_listener default/kubernetes:443 HTTPS 443 "$lb_name"
     create_load_balancer_pool default/kubernetes:443 HTTPS ROUND_ROBIN \
         default/kubernetes:443 "$lb_name"
@@ -642,6 +642,51 @@ function run_kuryr_daemon {
     run_process kuryr-daemon "$daemon_bin --config-file $KURYR_CONFIG" root root
 }
 
+function create_ingress_l7_router {
+
+    local lb_port_id
+    local lb_name
+    local project_id
+    local max_timeout
+    local lb_vip
+    local fake_svc_name
+    local l7_router_fip
+    local project_id
+
+    lb_name=${KURYR_L7_ROUTER_NAME}
+    max_timeout=600
+
+    create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"
+
+    wait_for_lb $lb_name $max_timeout
+
+    lb_port_id="$(get_loadbalancer_attribute "$lb_name" "vip_port_id")"
+
+    project_id=$(get_or_create_project \
+        "$KURYR_NEUTRON_DEFAULT_PROJECT" default)
+
+    #allocate FIP and bind it to lb vip
+    l7_router_fip=$(openstack --os-cloud devstack-admin \
+           --os-region "$REGION_NAME" \
+           floating ip create --project "$project_id" \
+            --subnet "${ext_svc_subnet_id}"  "${ext_svc_net_id}" \
+            -f value -c floating_ip_address)
+
+    openstack  --os-cloud devstack-admin \
+            --os-region "$REGION_NAME" \
+            floating ip set --port "$lb_port_id" "$l7_router_fip"
+
+    if is_service_enabled octavia; then
+        echo -n "Octavia: no need to create fake k8s service for Ingress."
+    else
+        # keep fake an endpoint less k8s service to keep Kubernetes API server
+        # from allocating ingress LB vip
+        fake_svc_name='kuryr-svc-ingress'
+        echo -n "LBaaS: create fake k8s service: $fake_svc_name for Ingress."
+        lb_vip="$(get_loadbalancer_attribute "$lb_name" "vip_address")"
+        create_k8s_fake_service $fake_svc_name $lb_vip
+    fi
+}
 
 source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes
 
@@ -776,6 +821,12 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
             create_k8s_router_fake_service
         fi
         create_k8s_api_service
+        #create Ingress L7 router if required
+        enable_ingress=$(trueorfalse False KURYR_ENABLE_INGRESS)
+
+        if [ "$enable_ingress" == "True" ]; then
+            create_ingress_l7_router
+        fi
 
         # FIXME(dulek): This is a very late phase to start Kuryr services.
         #               We're doing it here because we need K8s API LB to be
diff --git a/devstack/settings b/devstack/settings
index 2aa939b4a..97cc7c609 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -92,3 +92,9 @@ KURYR_CNI_HEALTH_SERVER_PORT=${KURYR_CNI_HEALTH_SERVER_PORT:-8090}
 # High availability of controller
 KURYR_CONTROLLER_HA_PORT=${KURYR_CONTROLLER_HA_PORT:-16401}
 KURYR_CONTROLLER_REPLICAS=${KURYR_CONTROLLER_REPLICAS:-1}
+
+# Kuryr ingress enable
+KURYR_ENABLE_INGRESS=${KURYR_ENABLE_INGRESS:-False}
+
+# Kuryr L7 router's name
+KURYR_L7_ROUTER_NAME=${KURYR_L7_ROUTER_NAME:-kuryr-l7-router}