From 324f4aca7d194991a028068a9b8acef6f74e11c9 Mon Sep 17 00:00:00 2001 From: Ton Ngo Date: Fri, 23 Sep 2016 19:51:28 +0000 Subject: [PATCH] Fix K8s load balancer with LBaaS v1 Fix node name and auth_url Update the url to Keystone v2 which has been changed. The name of the node registered in the kube-apiserver was also changed at some point to use the IP instead of the Nova instance name as was done originally, and this broke the Kubernetes plugin code. Change the node name back to the Nova instance name in the option --hostname-override for kubelet. Some update to the document. With this patch, the load balancer works with Magnum Newton and later, along with the image fedora-atomic-latest. Important notes: 1. The current image has Kubernetes release 1.2 and this only works with neutron LBaaS v1. Support for LBaaS v2 requires Kubernetes release 1.3 or later. Magnum support for 1.3 is still in development. 2. LBaaS v1 has been removed in Newton and is only available in Mitaka or by custom installation (likely requires some hacking). This means to get the load balancer feature, you will want to install Openstack Mitaka and Magnum Newton. Change-Id: Ica9d92c8d7410bf30832005687ecce4a90ef6c58 Closes-Bug: #1524025 --- doc/source/dev/kubernetes-load-balancer.rst | 15 ++++++++------- .../fragments/configure-kubernetes-master.sh | 2 +- .../fragments/configure-kubernetes-minion.sh | 12 +++++++++--- .../kubernetes/fragments/write-kube-os-config.sh | 2 +- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/doc/source/dev/kubernetes-load-balancer.rst b/doc/source/dev/kubernetes-load-balancer.rst index d1e3a16853..34360ecc01 100644 --- a/doc/source/dev/kubernetes-load-balancer.rst +++ b/doc/source/dev/kubernetes-load-balancer.rst @@ -109,6 +109,9 @@ perform the following steps: - --cloud_config=/etc/sysconfig/kube_openstack_config - --cloud_provider=openstack + When the file is saved, the pod will automatically restart the + kube-controller-manager container to pick up the change. + 3. Enter OpenStack user credential:: sudo vi /etc/sysconfig/kube_openstack_config @@ -124,12 +127,6 @@ perform the following steps: sudo service kube-apiserver restart service kube-apiserver status -5. Restart the kube-controller-manager container:: - - KUBE_CONTROLLER=$(sudo docker ps | awk '/k8s_kube-controller/{print $1}') - sudo docker restart $KUBE_CONTROLLER - sudo docker inspect $KUBE_CONTROLLER - This only needs to be done once. The steps can be reversed to disable the load balancer feature. Before deleting the Kubernetes cluster, make sure to delete all the services that created load balancers. Because the Neutron @@ -245,10 +242,14 @@ Finally associate the floating IP with the port of the VIP:: neutron floatingip-associate $FLOATING_ID $PORT_ID -The endpoint for nginx can now be accessed at this floating IP:: +The endpoint for nginx can now be accessed on a browser at this floating IP:: http://172.24.4.78:80 +Alternatively, you can check for the nginx 'welcome' message by:: + + curl http://172.24.4.78:80 + NOTE: it is not necessary to indicate port :80 here but it is shown to correlate with the port that was specified in the service manifest. diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 442dce144c..0d6308b585 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -45,7 +45,7 @@ _EOC_ sed -i ' /^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/ - /^KUBE_CONTROLLER_MANAGER_ARGS=/ s/KUBE_CONTROLLER_MANAGER_ARGS.*/#Uncomment the following line to enable Kubernetes Load Balancer feature \n#KUBE_CONTROLLER_MANAGER_ARGS="--cloud_config=\/etc\/sysconfig\/kube_openstack_config --cloud_provider=openstack"/ + /^KUBE_CONTROLLER_MANAGER_ARGS=/ s/KUBE_CONTROLLER_MANAGER_ARGS.*/#Uncomment the following line to enable Kubernetes Load Balancer feature \n#KUBE_CONTROLLER_MANAGER_ARGS="--cloud-config=\/etc\/sysconfig\/kube_openstack_config --cloud-provider=openstack"/ ' /etc/kubernetes/controller-manager KUBELET_ARGS="--register-node=true --register-schedulable=false --config=/etc/kubernetes/manifests --hostname-override=$KUBE_NODE_IP" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 4dcff6cedd..997b182d76 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -25,8 +25,14 @@ sed -i ' /^KUBE_MASTER=/ s|=.*|="--master='"$KUBE_MASTER_URI"'"| ' /etc/kubernetes/config +# NOTE: Kubernetes plugin for Openstack requires that the node name registered +# in the kube-apiserver be the same as the Nova name of the instance, so that +# the plugin can use the name to query for attributes such as IP, etc. +# The hostname of the node is set to be the Nova name of the instance, and +# the option --hostname-override for kubelet uses the hostname to register the node. +# Using any other name will break the load balancer and cinder volume features. HOSTNAME=$(hostname --short | sed 's/\.novalocal//') -KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 ${KUBE_CONFIG}" +KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 ${KUBE_CONFIG} --hostname-override=${HOSTNAME}" if [ -n "${INSECURE_REGISTRY_URL}" ]; then KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" @@ -37,7 +43,7 @@ sed -i ' /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ /^KUBELET_HOSTNAME=/ s/=.*/=""/ /^KUBELET_API_SERVER=/ s|=.*|="--api_servers='"$KUBE_MASTER_URI"'"| - /^KUBELET_ARGS=/ s|=.*|="'"${KUBELET_ARGS}"' --hostname-override='"${KUBE_NODE_IP}"'"| + /^KUBELET_ARGS=/ s|=.*|="'"${KUBELET_ARGS}"'"| ' /etc/kubernetes/kubelet sed -i ' @@ -75,7 +81,7 @@ tenant-name=$TENANT_NAME EOF cat << _EOC_ >> /etc/kubernetes/kubelet -#KUBELET_ARGS="$KUBELET_ARGS --hostname-override=${HOSTNAME} --cloud-provider=openstack --cloud-config=/etc/kubernetes/kube_openstack_config" +#KUBELET_ARGS="$KUBELET_ARGS --cloud-provider=openstack --cloud-config=/etc/kubernetes/kube_openstack_config" _EOC_ if [ ! -f /usr/bin/udevadm ]; then diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh index 8e3cfc66ed..c5b53145f0 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh @@ -5,7 +5,7 @@ KUBE_OS_CLOUD_CONFIG=/etc/sysconfig/kube_openstack_config # kubernetes backend only support keystone v2 at this point -AUTH_URL=$(echo "$AUTH_URL" | sed 's/v3/v2/') +AUTH_URL=$(echo "$AUTH_URL" | sed 's/v3/v2.0/') # Generate a the configuration for Kubernetes services # to talk to OpenStack Neutron