Merge "Enable multinode gate."

.zuul.d/base.yaml

@@ -86,6 +86,7 @@
         etcd3: true
         kubernetes-master: true
         kuryr-kubernetes: true
+        kuryr-daemon: true
         coredns: false
         neutron-tag-ports-during-bulk-creation: true
       zuul_copy_output:
@@ -158,6 +159,7 @@
         etcd3: true
         kubernetes-master: true
         kuryr-kubernetes: true
+        kuryr-daemon: true
         coredns: false
         neutron-tag-ports-during-bulk-creation: true
       zuul_copy_output:

.zuul.d/multinode.yaml

@@ -32,6 +32,7 @@
         devstack_services:
           # Need to disable dstat due to bug https://github.com/dstat-real/dstat/pull/162
           dstat: false
+          kuryr-daemon: true
           kuryr-kubernetes: false
           kubernetes-worker: true
           kubernetes-master: false
@@ -43,11 +44,14 @@
           USE_PYTHON3: true
     vars:
       devstack_localrc:
-          KURYR_K8S_API_URL: "http://${SERVICE_HOST}:${KURYR_K8S_API_PORT}"
+          KURYR_K8S_API_URL: "https://${SERVICE_HOST}:${KURYR_K8S_API_PORT}"
           KURYR_K8S_MULTI_WORKER_TESTS: True
       devstack_services:
         tls-proxy: false
-        kubernetes-worker: true
+        kubernetes-worker: false
+        kubernetes-master: true
+        kuryr-daemon: true
+        kuryr-kubernetes: true
     voting: false
 
 - job:

devstack/lib/kubernetes

@@ -124,6 +124,44 @@ EOF
     safe_chown $STACK_USER:$STACK_USER $kube_config_file
 }
 
+function kubeadm_join {
+    local output_dir="${DATA_DIR}/kuryr-kubernetes"
+
+    mkdir -p "${output_dir}"
+
+    cluster_ip_ranges=()
+    for service_subnet_id in ${KURYR_SERVICE_SUBNETS_IDS[@]}; do
+        service_cidr=$(openstack --os-cloud devstack-admin \
+            --os-region "$REGION_NAME" \
+            subnet show "$service_subnet_id" \
+            -c cidr -f value)
+        cluster_ip_ranges+=($(split_subnet "$service_cidr" | cut -f1))
+    done
+
+    # TODO(gryf): take care of cri-o case aswell
+    rm -f ${output_dir}/kubeadm-join.yaml
+    cat >> ${output_dir}/kubeadm-join.yaml << EOF
+apiVersion: kubeadm.k8s.io/v1beta2
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: ${SERVICE_HOST}:${KURYR_K8S_API_PORT}
+    token: "${KURYR_K8S_TOKEN}"
+    unsafeSkipCAVerification: true
+  tlsBootstrapToken: "${KURYR_K8S_TOKEN}"
+kind: JoinConfiguration
+nodeRegistration:
+  kubeletExtraArgs:
+    cgroup-driver: "$(docker info -f '{{.CgroupDriver}}')"
+    cni-bin-dir: "$CNI_BIN_DIR"
+    cni-conf-dir: "$CNI_CONF_DIR"
+    enable-server: "true"
+  taints:
+    []
+EOF
+    sudo -E kubeadm join --ignore-preflight-errors Swap \
+        --config ${output_dir}/kubeadm-join.yaml
+}
+
 function get_k8s_apiserver {
     # assumption is, there is no other cluster, so there is only one API
     # server.

devstack/lib/kuryr_kubernetes

@@ -272,11 +272,12 @@ function create_k8s_subnet {
     echo "$subnet_id"
 }
 
-# build_kuryr_containers
+# build_kuryr_container_image
-#   Description: Generates a Kuryr controller and Kuryr CNI docker images in
+#   Description: Generates a Kuryr controller or Kuryr CNI docker image in
-#       the local docker registry as kuryr/controller:latest and
+#       the local docker registry as kuryr/controller:latest for controller or
-#       kuryr/cni:latest respectively
+#       kuryr/cni:latest for CNI.
-function build_kuryr_containers {
+function build_kuryr_container_image {
+    local target=$1  # controller or cni
     local build_args
     local build_dir
 
@@ -293,11 +294,11 @@ function build_kuryr_containers {
     #               resolved instead of podman we need to use buildah directly,
     #               hence this awful if clause.
     if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
-        sudo buildah bud -t docker.io/kuryr/controller -f controller.Dockerfile .
+        sudo buildah bud -t "docker.io/kuryr/${target}" \
-        sudo buildah bud -t docker.io/kuryr/cni -f cni.Dockerfile .
+            -f "${target}.Dockerfile" .
     else
-        container_runtime build -t kuryr/controller -f controller.Dockerfile ${build_args} .
+        container_runtime build -t "kuryr/${target}" \
-        container_runtime build -t kuryr/cni -f cni.Dockerfile ${build_args} .
+            -f "${target}.Dockerfile" ${build_args} .
     fi
     popd
 }

devstack/plugin.sh

@@ -60,10 +60,9 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \
             build_install_kuryr_cni
         fi
 
-
     elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
         echo_summary "Configure kuryr bits"
-        if is_service_enabled kuryr-kubernetes; then
+        if is_service_enabled kuryr-daemon; then
             create_kuryr_account
             configure_kuryr
         fi
@@ -71,7 +70,7 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \
     elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
         echo_summary "Installing kubernetes and kuryr"
         # Initialize and start the template service
-        if is_service_enabled kubernetes-master; then
+        if is_service_enabled kuryr-kubernetes; then
             configure_neutron_defaults
         fi
 
@@ -79,14 +78,23 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \
             prepare_kubelet
         fi
 
-        if is_service_enabled kubernetes-master kubernetes-worker; then
+        if is_service_enabled kubernetes-master; then
             wait_for "etcd" "http://${SERVICE_HOST}:${ETCD_PORT}/v2/machines"
             kubeadm_init
             copy_kuryr_certs
         fi
 
+        if is_service_enabled kubernetes-worker; then
+            kubeadm_join
+        fi
+
         if [ "${KURYR_CONT}" == "True" ]; then
-            build_kuryr_containers
+            if is_service_enabled kubernetes-master; then
+                build_kuryr_container_image "controller"
+                build_kuryr_container_image "cni"
+            else
+                build_kuryr_container_image "cni"
+            fi
         fi
 
         if is_service_enabled kubernetes-master; then
@@ -120,21 +128,23 @@ if is_service_enabled kuryr-kubernetes kuryr-daemon \
 
     elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
         echo_summary "Run kuryr-kubernetes"
-        if is_service_enabled octavia; then
+        if is_service_enabled kuryr-kubernetes; then
-            create_lb_for_services
+            if is_service_enabled octavia; then
-        fi
+                create_lb_for_services
+            fi
 
-        # FIXME(dulek): This is a very late phase to start Kuryr services.
+            # FIXME(dulek): This is a very late phase to start Kuryr services.
-        #               We're doing it here because we need K8s API LB to be
+            # We're doing it here because we need K8s API LB to be created in
-        #               created in order to run kuryr services. Thing is
+            # order to run kuryr services. Thing is Octavia is unable to
-        #               Octavia is unable to create LB until test-config phase.
+            # create LB until test-config phase. We can revisit this once
-        #               We can revisit this once Octavia's DevStack plugin will
+            # Octavia's DevStack plugin will get improved.
-        #               get improved.
+
-        if [ "${KURYR_CONT}" == "True" ]; then
+            if [ "${KURYR_CONT}" == "True" ]; then
-            run_containerized_kuryr_resources
+                run_containerized_kuryr_resources
-        else
+            else
-            run_kuryr_kubernetes
+                run_kuryr_kubernetes
-            run_kuryr_daemon
+                run_kuryr_daemon
+            fi
         fi
 
         if is_service_enabled tempest; then