From c0787b4e94f68bd52088610666d1d17cfe4e53bf Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 13 Mar 2017 18:33:47 +0100 Subject: [PATCH] Fix usage of the trustee user in K8S Cinder plugin Closes-Bug: #1672667 Change-Id: I702818777ea4664ecd560c4b7a02431c86988e17 --- doc/source/dev/kubernetes-load-balancer.rst | 55 ++----------------- doc/source/userguide.rst | 51 +---------------- .../fragments/configure-kubernetes-master.sh | 24 ++++---- .../fragments/configure-kubernetes-minion.sh | 44 ++++----------- .../fragments/write-heat-params-master.yaml | 1 + .../fragments/write-heat-params.yaml | 2 - .../fragments/write-kube-os-config.sh | 9 +-- magnum/drivers/heat/template_def.py | 13 +---- .../templates/kubecluster.yaml | 1 + .../templates/kubemaster.yaml | 5 ++ .../templates/kubeminion.yaml | 9 ++- .../handlers/test_swarm_cluster_conductor.py | 2 +- 12 files changed, 49 insertions(+), 167 deletions(-) diff --git a/doc/source/dev/kubernetes-load-balancer.rst b/doc/source/dev/kubernetes-load-balancer.rst index 34360ecc01..c744393c7a 100644 --- a/doc/source/dev/kubernetes-load-balancer.rst +++ b/doc/source/dev/kubernetes-load-balancer.rst @@ -78,57 +78,7 @@ neutron_lbaas.conf:: To configure LBaaS v1 or v2, refer to the Neutron documentation. -To enable the load balancer, log into each master node of your cluster and -perform the following steps: - -1. Configure kube-apiserver:: - - sudo vi /etc/kubernetes/apiserver - - Comment out the line:: - - #KUBE_API_ARGS="--runtime_config=api/all=true" - - Uncomment the line:: - - KUBE_API_ARGS="--runtime_config=api/all=true --cloud_config=/etc/sysconfig/kube_openstack_config --cloud_provider=openstack""" - -2. Configure kube-controller-manager:: - - sudo vi /etc/kubernetes/manifests/kube-controller-manager.yaml - - Immediately after the lines:: - - - controller-manager - - --master=http://127.0.0.1:8080 - - --service-account-private-key-file=/etc/kubernetes/ssl/server.key - - --root-ca-file=/etc/kubernetes/ssl/ca.crt - - Add the following lines:: - - - --cloud_config=/etc/sysconfig/kube_openstack_config - - --cloud_provider=openstack - - When the file is saved, the pod will automatically restart the - kube-controller-manager container to pick up the change. - -3. Enter OpenStack user credential:: - - sudo vi /etc/sysconfig/kube_openstack_config - - The username and tenant-name entries have been filled in with the - Keystone values of the user who created the cluster. Enter the password - of this user on the entry for password:: - - password=ChangeMe - -4. Restart the Kubernetes API server:: - - sudo service kube-apiserver restart - service kube-apiserver status - -This only needs to be done once. The steps can be reversed to disable the -load balancer feature. Before deleting the Kubernetes cluster, make sure to +Before deleting the Kubernetes cluster, make sure to delete all the services that created load balancers. Because the Neutron objects created by Kubernetes are not managed by Heat, they will not be deleted by Heat and this will cause the cluster-delete operation to fail. If @@ -138,6 +88,9 @@ lb-healthmonitor) and then run cluster-delete again. Steps for the users =================== +This feature requires the OpenStack cloud provider to be enabled. +To do so, enable the cinder support (--volume-driver cinder). + For the user, publishing the service endpoint externally involves the following 2 steps: diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index 1e4f81253f..24d8a31c00 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -2212,12 +2212,8 @@ Following are some examples for using Cinder as persistent storage. Using Cinder in Kubernetes ++++++++++++++++++++++++++ -**NOTE:** This feature requires Kubernetes version 1.1.1 or above and -Docker version 1.8.3 or above. The public Fedora image from Atomic -currently meets this requirement. - -**NOTE:** The following steps are a temporary workaround, and Magnum's -development team is working on a long term solution to automate these steps. +**NOTE:** This feature requires Kubernetes version 1.5.0 or above. +The public Fedora image from Atomic currently meets this requirement. 1. Create the ClusterTemplate. @@ -2240,49 +2236,6 @@ development team is working on a long term solution to automate these steps. --cluster-template k8s-cluster-template \ --node-count 1 - -3. Configure kubelet. - - To allow Kubernetes to interface with Cinder, log into each minion - node of your cluster and perform step 4 through 6:: - - sudo vi /etc/kubernetes/kubelet - - Comment out the line:: - - #KUBELET_ARGS=--config=/etc/kubernetes/manifests --cadvisor-port=4194 - - Uncomment the line:: - - #KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 --cloud-provider=openstack --cloud-config=/etc/kubernetes/kube_openstack_config" - - -4. Enter OpenStack user credential:: - - sudo vi /etc/kubernetes/kube_openstack_config - - The username, tenant-name and region entries have been filled in with the - Keystone values of the user who created the cluster. Enter the password - of this user on the entry for password:: - - password=ChangeMe - -5. Restart Kubernetes services:: - - sudo systemctl restart kubelet - - On restart, the new configuration enables the Kubernetes cloud provider - plugin for OpenStack, along with the necessary credential for kubelet - to authenticate with Keystone and to make request to OpenStack services. - -6. Install nsenter:: - - sudo docker run -v /usr/local/bin:/target jpetazzo/nsenter - - The nsenter utility is used by Kubernetes to run new processes within - existing kernel namespaces. This allows the kubelet agent to manage storage - for pods. - Kubernetes is now ready to use Cinder for persistent storage. Following is an example illustrating how Cinder is used in a pod. diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 3a0232be5e..c6de5b2407 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -4,11 +4,6 @@ echo "configuring kubernetes (master)" -if [ -z "$KUBE_NODE_IP" ]; then - # FIXME(yuanying): Set KUBE_NODE_IP correctly - KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) -fi - sed -i ' /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/ ' /etc/kubernetes/config @@ -31,6 +26,10 @@ if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_ADMISSION_CONTROL="--admission-control=${ADMISSION_CONTROL_LIST}" fi +if [ -n "$TRUST_ID" ]; then + KUBE_API_ARGS="$KUBE_API_ARGS --cloud-config=/etc/sysconfig/kube_openstack_config --cloud-provider=openstack" +fi + sed -i ' /^KUBE_API_ADDRESS=/ s/=.*/="'"${KUBE_API_ADDRESS}"'"/ /^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"$PORTAL_NETWORK_CIDR"'"| @@ -39,10 +38,7 @@ sed -i ' /^KUBE_ADMISSION_CONTROL=/ s/=.*/="'"${KUBE_ADMISSION_CONTROL}"'"/ ' /etc/kubernetes/apiserver cat << _EOC_ >> /etc/kubernetes/apiserver -#Uncomment the following line to disable Load Balancer feature KUBE_API_ARGS="$KUBE_API_ARGS" -#Uncomment the following line to enable Load Balancer feature -#KUBE_API_ARGS="$KUBE_API_ARGS --cloud-config=/etc/sysconfig/kube_openstack_config --cloud-provider=openstack" _EOC_ # Add controller manager args @@ -50,16 +46,18 @@ KUBE_CONTROLLER_MANAGER_ARGS="" if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/srv/kubernetes/server.key --root-ca-file=/srv/kubernetes/ca.crt" fi + +if [ -n "$TRUST_ID" ]; then + KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/sysconfig/kube_openstack_config --cloud-provider=openstack" +fi + sed -i ' /^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/ /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"# ' /etc/kubernetes/controller-manager -cat << _EOC_ >> /etc/kubernetes/controller-manager -#Uncomment the following line to enable Kubernetes Load Balancer feature -#KUBE_CONTROLLER_MANAGER_ARGS="\$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/sysconfig/kube_openstack_config --cloud-provider=openstack" -_EOC_ -KUBELET_ARGS="--register-node=true --register-schedulable=false --config=/etc/kubernetes/manifests --hostname-override=$KUBE_NODE_IP" +HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') +KUBELET_ARGS="--register-node=true --register-schedulable=false --config=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}" if [ -n "${INSECURE_REGISTRY_URL}" ]; then KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh index d48cbb4cbe..3e50cba841 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -4,11 +4,6 @@ echo "configuring kubernetes (minion)" -if [ -z "$KUBE_NODE_IP" ]; then - # FIXME(yuanying): Set KUBE_NODE_IP correctly - KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) -fi - CERT_DIR=/srv/kubernetes PROTOCOL=https FLANNEL_OPTIONS="-etcd-cafile $CERT_DIR/ca.crt \ @@ -52,8 +47,17 @@ sed -i ' # The hostname of the node is set to be the Nova name of the instance, and # the option --hostname-override for kubelet uses the hostname to register the node. # Using any other name will break the load balancer and cinder volume features. -HOSTNAME=$(hostname --short | sed 's/\.novalocal//') -KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 ${KUBE_CONFIG} --hostname-override=${HOSTNAME}" +HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//') +KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 ${KUBE_CONFIG} --hostname-override=${HOSTNAME_OVERRIDE}" + +if [ -n "$TRUST_ID" ]; then + KUBELET_ARGS="$KUBELET_ARGS --cloud-provider=openstack --cloud-config=/etc/sysconfig/kube_openstack_config" +fi + +# Workaround for Cinder support (fixed in k8s >= 1.6) +if [ ! -f /usr/bin/udevadm ]; then + ln -s /sbin/udevadm /usr/bin/udevadm +fi if [ -n "${INSECURE_REGISTRY_URL}" ]; then KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" @@ -86,32 +90,6 @@ if [ "$NETWORK_DRIVER" = "flannel" ]; then done fi -if [ "$VOLUME_DRIVER" = "cinder" ]; then - CLOUD_CONFIG=/etc/kubernetes/kube_openstack_config - KUBERNETES=/etc/kubernetes - if [ ! -d ${KUBERNETES} -o ! -f ${CLOUD_CONFIG} ]; then - mkdir -p $KUBERNETES - fi - AUTH_URL=${AUTH_URL/v3/v2.0} -cat > $CLOUD_CONFIG <> /etc/kubernetes/kubelet -#KUBELET_ARGS="$KUBELET_ARGS --cloud-provider=openstack --cloud-config=/etc/kubernetes/kube_openstack_config" -_EOC_ - - if [ ! -f /usr/bin/udevadm ]; then - ln -s /sbin/udevadm /usr/bin/udevadm - fi - -fi - cat >> /etc/environment < $KUBE_OS_CLOUD_CONFIG <