Use K8s API LB IP instead of HOST_IP in DevStack

This commit switches Kuryr configuration to use the IP of load balancer
created for Kubernetes API instead of hardcoding HOST_IP there. This
should be useful e.g. for multinode.

Change-Id: I4535986fdac4d7d6b2eda0debdcaf6122a4f5b9f
Closes-Bug: 1771801

devstack/lib/kuryr_kubernetes

@@ -748,7 +748,6 @@ function run_openshift_master {
         "--write-config=${OPENSHIFT_DATA_DIR}"
 
     # Reconfigure Kuryr-Kubernetes to use the certs generated
-    iniset "$KURYR_CONFIG" kubernetes api_root "$OPENSHIFT_API_URL"
     iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "${OPENSHIFT_DATA_DIR}/admin.crt"
     iniset "$KURYR_CONFIG" kubernetes ssl_client_key_file "${OPENSHIFT_DATA_DIR}/admin.key"
     iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "${OPENSHIFT_DATA_DIR}/ca.crt"

devstack/plugin.sh

@@ -52,7 +52,6 @@ function configure_kuryr {
     sudo install -o "$STACK_USER" -m 640 -D "${KURYR_HOME}/etc/kuryr.conf.sample" \
         "$KURYR_CONFIG"
 
-    iniset "$KURYR_CONFIG" kubernetes api_root "$KURYR_K8S_API_URL"
     if [ "$KURYR_K8S_API_CERT" ]; then
         iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "$KURYR_K8S_API_CERT"
     fi
@@ -61,6 +60,7 @@ function configure_kuryr {
     fi
     if [ "$KURYR_K8S_API_CACERT" ]; then
         iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "$KURYR_K8S_API_CACERT"
+        iniset "$KURYR_CONFIG" kubernetes ssl_verify_server_crt True
     fi
     # REVISIT(ivc): 'use_stderr' is required for current CNI driver. Once a
     # daemon-based CNI driver is implemented, this could be removed.
@@ -73,6 +73,9 @@ function configure_kuryr {
     iniset "$KURYR_CONFIG" kubernetes pod_subnets_driver "$KURYR_SUBNET_DRIVER"
     iniset "$KURYR_CONFIG" kubernetes enabled_handlers "$KURYR_ENABLED_HANDLERS"
 
+    # Let Kuryr retry connections to K8s API for 20 minutes.
+    iniset "$KURYR_CONFIG" kubernetes watch_retry_timeout 1200
+
     KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
     if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
         # This works around the issue of being unable to set oslo.privsep mode
@@ -237,9 +240,9 @@ function create_k8s_api_service {
 
     create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
             "$project_id" "$k8s_api_clusterip"
-    create_load_balancer_listener default/kubernetes:443 HTTPS 443 "$lb_name" "$project_id" 3600000
+    create_load_balancer_listener default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ${KURYR_K8S_API_LB_PORT} "$lb_name" "$project_id" 3600000
-    create_load_balancer_pool default/kubernetes:443 HTTPS ROUND_ROBIN \
+    create_load_balancer_pool default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ROUND_ROBIN \
-        default/kubernetes:443 "$project_id" "$lb_name"
+        default/kubernetes:${KURYR_K8S_API_LB_PORT} "$project_id" "$lb_name"
 
     local api_port
     if is_service_enabled openshift-master; then
@@ -260,10 +263,10 @@ function create_k8s_api_service {
     if [[ "$use_octavia" == "True" && \
           "$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L2" ]]; then
         create_load_balancer_member "$(hostname)" "$address" "$api_port" \
-            default/kubernetes:443 $KURYR_NEUTRON_DEFAULT_POD_SUBNET "$lb_name" "$project_id"
+            default/kubernetes:${KURYR_K8S_API_LB_PORT} $KURYR_NEUTRON_DEFAULT_POD_SUBNET "$lb_name" "$project_id"
     else
         create_load_balancer_member "$(hostname)" "$address" "$api_port" \
-            default/kubernetes:443 public-subnet "$lb_name" "$project_id"
+            default/kubernetes:${KURYR_K8S_API_LB_PORT} public-subnet "$lb_name" "$project_id"
     fi
 }
 
@@ -357,6 +360,22 @@ function configure_neutron_defaults {
         sg_ids+=",${octavia_pod_access_sg_id}"
     fi
 
+    KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
+    if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
+        local service_cidr
+        local k8s_api_clusterip
+        service_cidr=$(openstack --os-cloud devstack-admin \
+                                 --os-region "$REGION_NAME" \
+                                 subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
+                                 -c cidr -f value)
+        k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
+        # NOTE(dulek): KURYR_K8S_API_LB_URL will be a global to be used by next
+        #              deployment phases.
+        KURYR_K8S_API_LB_URL="https://${k8s_api_clusterip}:${KURYR_K8S_API_LB_PORT}"
+        iniset "$KURYR_CONFIG" kubernetes api_root ${KURYR_K8S_API_LB_URL}
+    else
+        iniset "$KURYR_CONFIG" kubernetes api_root ""
+    fi
     iniset "$KURYR_CONFIG" neutron_defaults project "$project_id"
     iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "$pod_subnet_id"
     iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups "$sg_ids"
@@ -435,6 +454,11 @@ function prepare_kubernetes_files {
     sudo bash -c "echo '$(create_token),kubelet,kubelet' >> ${KURYR_HYPERKUBE_DATA_DIR}/known_tokens.csv"
     sudo bash -c "echo '$(create_token),kube_proxy,kube_proxy' >> ${KURYR_HYPERKUBE_DATA_DIR}/known_tokens.csv"
 
+    # Copy certs for Kuryr services to use
+    sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/kubecfg.crt" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"
+    sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/kubecfg.key" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"
+    sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/ca.crt" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"
+
     # FIXME(ivc): replace 'sleep' with a strict check (e.g. wait_for_files)
     # 'kubernetes-api' fails if started before files are generated.
     # this is a workaround to prevent races.
@@ -456,7 +480,7 @@ function wait_for {
     extra_flags=${cacert_path:+"--cacert ${cacert_path}"}
 
     local start_time=$(date +%s)
-    until curl -o /dev/null -sf $extra_flags "$url"; do
+    until curl -o /dev/null -s $extra_flags "$url"; do
         echo -n "."
         local curr_time=$(date +%s)
         local time_diff=$(($curr_time - $start_time))
@@ -631,13 +655,14 @@ function run_k8s_kubelet {
 
 function run_kuryr_kubernetes {
     local python_bin=$(which python)
-
     if is_service_enabled openshift-master; then
-        wait_for "OpenShift API Server" "$OPENSHIFT_API_URL" \
+        wait_for "OpenShift API Server" "$KURYR_K8S_API_LB_URL" \
-            "${OPENSHIFT_DATA_DIR}/ca.crt"
+            "${OPENSHIFT_DATA_DIR}/ca.crt" 1200
     else
-        wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
+        wait_for "Kubernetes API Server" "$KURYR_K8S_API_LB_URL" \
+            "${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt" 1200
     fi
+
     run_process kuryr-kubernetes \
         "$python_bin ${KURYR_HOME}/scripts/run_server.py  \
             --config-file $KURYR_CONFIG"
@@ -791,12 +816,6 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
         run_k8s_scheduler
     fi
 
-    KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
-    if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
-        # If running in containerized mode, we'll run the daemon as DaemonSet.
-        run_kuryr_daemon
-    fi
-
     if is_service_enabled kubelet; then
         prepare_kubelet
         extract_hyperkube
@@ -813,6 +832,7 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
         configure_k8s_pod_sg_rules
     fi
 
+    KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
     if is_service_enabled kuryr-kubernetes; then
         /usr/local/bin/kubectl apply -f ${KURYR_HOME}/kubernetes_crds/kuryrnet.yaml
         if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
@@ -831,7 +851,6 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
     if is_service_enabled kuryr-kubernetes; then
         # NOTE(dulek): This is so late, because Devstack's Octavia is unable
         #              to create loadbalancers until test-config phase.
-        local use_octavia
         use_octavia=$(trueorfalse True KURYR_K8S_LBAAS_USE_OCTAVIA)
         if [[ "$use_octavia" == "False" ]]; then
             create_k8s_router_fake_service
@@ -846,7 +865,7 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
 
         # FIXME(dulek): This is a very late phase to start Kuryr services.
         #               We're doing it here because we need K8s API LB to be
-        #               created in order to run kuryr-cni container. Thing is
+        #               created in order to run kuryr services. Thing is
         #               Octavia is unable to create LB until test-config phase.
         #               We can revisit this once Octavia's DevStack plugin will
         #               get improved.
@@ -854,6 +873,7 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
             run_containerized_kuryr_resources
         else
             run_kuryr_kubernetes
+            run_kuryr_daemon
         fi
     fi
     if is_service_enabled tempest && [[ "$KURYR_USE_PORT_POOLS" == "True" ]]; then

devstack/settings

@@ -40,10 +40,12 @@ KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube}
 
 # Kubernetes
 KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080}
-KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-http://${HOST_IP}:${KURYR_K8S_API_PORT}}
+# NOTE(dulek): [kubernetes]api_root option will use LB IP instead.
-KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-}
+KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-"http://${HOST_IP}:${KURYR_K8S_API_PORT}"}
-KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-}
+KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"}
-KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-}
+KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"}
+KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"}
+KURYR_K8S_API_LB_PORT=${KURYR_K8S_API_LB_PORT:-443}
 KURYR_PORT_DEBUG=${KURYR_PORT_DEBUG:-True}
 KURYR_SUBNET_DRIVER=${KURYR_SUBNET_DRIVER:-default}
 KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,lb,lbaasspec}

kuryr_kubernetes/clients.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+
 from kuryr.lib import utils
 
 from kuryr_kubernetes import config
@@ -68,5 +70,12 @@ def setup_loadbalancer_client():
 
 
 def setup_kubernetes_client():
-    _clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(
+    if config.CONF.kubernetes.api_root:
-        config.CONF.kubernetes.api_root)
+        api_root = config.CONF.kubernetes.api_root
+    else:
+        # NOTE(dulek): This is for containerized deployments, i.e. running in
+        #              K8s Pods.
+        host = os.environ['KUBERNETES_SERVICE_HOST']
+        port = os.environ['KUBERNETES_SERVICE_PORT_HTTPS']
+        api_root = "https://%s:%s" % (host, port)
+    _clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(api_root)