OCP-Router: add support for external Ingress Controller's LB creation

This is the first patch of the Ingress Controller capability.

In order for the K8S Ingress and OpenShift Route resources to work,
the cluster must have an Ingress Controller running.
The Kuryr's Ingress Controller implementation will be based on
Octavia L7 load balancing.

This patch adds support for the creation of an external Load Balancer
in Devstack deployment, the follow-up Ingress Controller patches will
configure the L7 rules in that LB to perform the actual L7 routing.

Partially Implements: blueprint openshift-router-support
Change-Id: I9c18bd1d2d0f2127a1a924efe7976a38b6f7cc51

devstack/lib/kuryr_kubernetes

@@ -316,10 +316,23 @@ function create_k8s_router_fake_service {
         --os-region "$REGION_NAME" \
         subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
         -f value -c gateway_ip)
+    create_k8s_fake_service $fake_svc_name $router_ip
+}
+
+# create_k8s_fake_service
+#   Description: Creates an endpoint-less kubernetes service to keep Kubernetes
+#                API server from allocating this IP for another service
+function create_k8s_fake_service {
+    local svc_name
+    local svc_ip
+
+    fake_svc_name="$1"
+    fake_svc_ip="$2"
+
     existing_svc_ip=$(/usr/local/bin/kubectl get svc --namespace kube-system -o jsonpath='{.items[?(@.metadata.name=='"\"${fake_svc_name}\""')].spec.clusterIP}')
 
     if [[ "$existing_svc_ip" == "" ]]; then
-        # Create fake router service so the router clusterIP can't be reassigned
+        # Create fake service so the clusterIP can't be reassigned
         cat <<EOF | /usr/local/bin/kubectl create -f -
 kind: Service
 apiVersion: v1
@@ -328,7 +341,7 @@ metadata:
   namespace: kube-system
 spec:
   type: ClusterIP
-  clusterIP: "${router_ip}"
+  clusterIP: "${fake_svc_ip}"
   ports:
     - protocol: TCP
       port: 80
@@ -852,25 +865,30 @@ function wait_for_lb {
 #                or Octavia
 #   Params:
 #       lb_name: Name to give to the load balancer.
-#       lb_vip: Virtual IP to give to the load balancer.
 #       lb_vip_subnet: Id or name of the subnet where lb_vip should be
 #                      allocated.
+#       lb_vip: Virtual IP to give to the load balancer - optional.
 function create_load_balancer {
     local lb_name
-    local lb_vip
     local lb_vip_subnet
+    local lb_params
 
     lb_name="$1"
-    lb_vip="$2"
-    lb_vip_subnet="$3"
-    if is_service_enabled octavia; then
-        openstack loadbalancer create --name "$lb_name" \
-            --vip-address "$lb_vip" \
-            --vip-subnet-id "$lb_vip_subnet"
+    lb_vip_subnet="$2"
+
+    lb_params=" --name $lb_name "
+    if [ -z "$3" ]; then
+        echo -n "create_load_balancer LB=$lb_name, lb_vip not provided."
     else
-        neutron lbaas-loadbalancer-create --name "$lb_name" \
-            --vip-address "$lb_vip" \
-            "$lb_vip_subnet"
+        lb_params+=" --vip-address $3"
+    fi
+
+    if is_service_enabled octavia; then
+        lb_params+=" --vip-subnet-id $lb_vip_subnet"
+        openstack loadbalancer create $lb_params
+    else
+        lb_params+=" $lb_vip_subnet"
+        neutron lbaas-loadbalancer-create $lb_params
     fi
 }
 
@@ -1021,3 +1039,22 @@ first, last = n.subnets(prefixlen_diff=1)
 print("%s\\t%s" % (first, last))
 EOF
 }
+
+# get_loadbalancer_attribute
+#   Description: Get load balancer attribute
+#   Params:
+#       lb_name: Load balancer name
+#       lb_attr: attribute name
+function get_loadbalancer_attribute {
+    local lb_name
+    local lb_attr
+
+    lb_name="$1"
+    lb_attr="$2"
+
+    if is_service_enabled octavia; then
+        openstack loadbalancer show "$lb_name" -c "$lb_attr" -f value
+    else
+        neutron lbaas-loadbalancer-show "$lb_name" -c "$lb_attr" -f value
+    fi
+}

devstack/local.conf.openshift.sample

@@ -160,6 +160,20 @@ enable_service kuryr-kubernetes
 # Since Rocky release this is a default deployment configuration.
 enable_service kuryr-daemon
 
+# Kuryr enable L7 routing
+# ========================
+#
+# Uncomment the next line to enable the L7 Routing/Ingress controller
+#
+#KURYR_ENABLE_INGRESS=True
+
+# Kuryr L7 router/lb name
+# ========================
+#
+# Edit the next line to change L7 Router/LB name
+#
+#KURYR_L7_ROUTER_NAME=kuryr-l7-router
+
 # Containerized Kuryr
 # ===================
 #

devstack/plugin.sh

@@ -232,8 +232,8 @@ function create_k8s_api_service {
 
     k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
 
-    create_load_balancer "$lb_name" "$k8s_api_clusterip" \
-        "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"
+    create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
+            "$k8s_api_clusterip"
     create_load_balancer_listener default/kubernetes:443 HTTPS 443 "$lb_name"
     create_load_balancer_pool default/kubernetes:443 HTTPS ROUND_ROBIN \
         default/kubernetes:443 "$lb_name"
@@ -642,6 +642,51 @@ function run_kuryr_daemon {
     run_process kuryr-daemon "$daemon_bin --config-file $KURYR_CONFIG" root root
 }
 
+function create_ingress_l7_router {
+
+    local lb_port_id
+    local lb_name
+    local project_id
+    local max_timeout
+    local lb_vip
+    local fake_svc_name
+    local l7_router_fip
+    local project_id
+
+    lb_name=${KURYR_L7_ROUTER_NAME}
+    max_timeout=600
+
+    create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"
+
+    wait_for_lb $lb_name $max_timeout
+
+    lb_port_id="$(get_loadbalancer_attribute "$lb_name" "vip_port_id")"
+
+    project_id=$(get_or_create_project \
+        "$KURYR_NEUTRON_DEFAULT_PROJECT" default)
+
+    #allocate FIP and bind it to lb vip
+    l7_router_fip=$(openstack --os-cloud devstack-admin \
+           --os-region "$REGION_NAME" \
+           floating ip create --project "$project_id" \
+            --subnet "${ext_svc_subnet_id}"  "${ext_svc_net_id}" \
+            -f value -c floating_ip_address)
+
+    openstack  --os-cloud devstack-admin \
+            --os-region "$REGION_NAME" \
+            floating ip set --port "$lb_port_id" "$l7_router_fip"
+
+    if is_service_enabled octavia; then
+        echo -n "Octavia: no need to create fake k8s service for Ingress."
+    else
+        # keep fake an endpoint less k8s service to keep Kubernetes API server
+        # from allocating ingress LB vip
+        fake_svc_name='kuryr-svc-ingress'
+        echo -n "LBaaS: create fake k8s service: $fake_svc_name for Ingress."
+        lb_vip="$(get_loadbalancer_attribute "$lb_name" "vip_address")"
+        create_k8s_fake_service $fake_svc_name $lb_vip
+    fi
+}
 
 source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes
 
@@ -776,6 +821,12 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
             create_k8s_router_fake_service
         fi
         create_k8s_api_service
+        #create Ingress L7 router if required
+        enable_ingress=$(trueorfalse False KURYR_ENABLE_INGRESS)
+
+        if [ "$enable_ingress" == "True" ]; then
+            create_ingress_l7_router
+        fi
 
         # FIXME(dulek): This is a very late phase to start Kuryr services.
         #               We're doing it here because we need K8s API LB to be

devstack/settings

@@ -92,3 +92,9 @@ KURYR_CNI_HEALTH_SERVER_PORT=${KURYR_CNI_HEALTH_SERVER_PORT:-8090}
 # High availability of controller
 KURYR_CONTROLLER_HA_PORT=${KURYR_CONTROLLER_HA_PORT:-16401}
 KURYR_CONTROLLER_REPLICAS=${KURYR_CONTROLLER_REPLICAS:-1}
+
+# Kuryr ingress enable
+KURYR_ENABLE_INGRESS=${KURYR_ENABLE_INGRESS:-False}
+
+# Kuryr L7 router's name
+KURYR_L7_ROUTER_NAME=${KURYR_L7_ROUTER_NAME:-kuryr-l7-router}