[k8s][fedora atomic] Rolling upgrade support

Rolling ugprade is an important feature for a managed k8s service,
at this stage, two user cases will be covered:

1. Upgrade base operating system
2. Upgrade k8s version

Known limitation: When doing operating system upgrade, there is no
chance to call kubectl drain to evict pods on that node.

Task: 30185
Story: 2002210

Change-Id: Ibbed59bc135969174a20e5243ff8464908801a23

magnum/api/controllers/v1/cluster_actions.py

@@ -151,6 +151,7 @@ class ActionsController(base.Controller):
         else:
             nodegroup = objects.NodeGroup.get(
                 context, cluster.uuid, cluster_upgrade_req.nodegroup)
+
         pecan.request.rpcapi.cluster_upgrade(
             cluster,
             new_cluster_template,

magnum/drivers/common/templates/fragments/configure-docker-registry.sh

@@ -2,11 +2,9 @@
 
 . /etc/sysconfig/heat-params
 
-if [ "$REGISTRY_ENABLED" = "False" ]; then
+if [ "$REGISTRY_ENABLED" = "True" ]; then
-    exit 0
+    ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
-fi
+    $ssh_cmd cat > /etc/sysconfig/registry-config.yml << EOF
-
-cat > /etc/sysconfig/registry-config.yml << EOF
 version: 0.1
 log:
   fields:
@@ -28,7 +26,7 @@ http:
     addr: :5000
 EOF
 
-cat > /etc/systemd/system/registry.service << EOF
+    $ssh_cmd cat > /etc/systemd/system/registry.service << EOF
 [Unit]
 Description=Docker registry v2
 Requires=docker.service
@@ -43,3 +41,5 @@ ExecStop=/usr/bin/docker rm -f registry
 [Install]
 WantedBy=multi-user.target
 EOF
+
+fi

magnum/drivers/common/templates/fragments/configure-docker-storage.sh

@@ -2,22 +2,24 @@
 
 . /etc/sysconfig/heat-params
 
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then
     if [ "$ENABLE_CINDER" == "False" ]; then
         # FIXME(yuanying): Use ephemeral disk for docker storage
         # Currently Ironic doesn't support cinder volumes,
         # so we must use preserved ephemeral disk instead of a cinder volume.
-        device_path=$(readlink -f /dev/disk/by-label/ephemeral0)
+        device_path=$($ssh_cmd readlink -f /dev/disk/by-label/ephemeral0)
     else
         attempts=60
         while [ ${attempts} -gt 0 ]; do
-            device_name=$(ls /dev/disk/by-id | grep ${DOCKER_VOLUME:0:20}$)
+            device_name=$($ssh_cmd ls /dev/disk/by-id | grep ${DOCKER_VOLUME:0:20}$)
             if [ -n "${device_name}" ]; then
                 break
             fi
             echo "waiting for disk device"
             sleep 0.5
-            udevadm trigger
+            $ssh_cmd udevadm trigger
             let attempts--
         done
 

magnum/drivers/common/templates/fragments/configure_docker_storage_driver_atomic.sh

@@ -4,12 +4,14 @@
 # * Remove any existing docker-storage configuration. In case of an
 #   existing configuration, docker-storage-setup will fail.
 # * Remove docker storage graph
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 clear_docker_storage () {
     # stop docker
-    systemctl stop docker
+    $ssh_cmd systemctl stop docker
-    systemctl disable docker-storage-setup
+    $ssh_cmd systemctl disable docker-storage-setup
     # clear storage graph
-    rm -rf /var/lib/docker/*
+    $ssh_cmd rm -rf /var/lib/docker/*
 
     if [ -f /etc/sysconfig/docker-storage ]; then
         sed -i "/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=/" /etc/sysconfig/docker-storage
@@ -21,9 +23,9 @@ configure_storage_driver_generic() {
     clear_docker_storage
 
     if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then
-        mkfs.xfs -f ${device_path}
+        $ssh_cmd mkfs.xfs -f ${device_path}
         echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab
-        mount -a
+        $ssh_cmd mount -a
     fi
 
     echo "DOCKER_STORAGE_OPTIONS=\"--storage-driver $1\"" > /etc/sysconfig/docker-storage
@@ -38,8 +40,8 @@ configure_devicemapper () {
 
     if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then
 
-        pvcreate -f ${device_path}
+        $ssh_cmd pvcreate -f ${device_path}
-        vgcreate docker ${device_path}
+        $ssh_cmd vgcreate docker ${device_path}
 
         echo "VG=docker" >> /etc/sysconfig/docker-storage-setup
     else
@@ -47,5 +49,5 @@ configure_devicemapper () {
         echo "DATA_SIZE=95%FREE" >> /etc/sysconfig/docker-storage-setup
     fi
 
-    docker-storage-setup
+    $ssh_cmd docker-storage-setup
 }

magnum/drivers/common/templates/fragments/enable-docker-registry.sh

@@ -6,7 +6,9 @@ if [ "$REGISTRY_ENABLED" = "False" ]; then
     exit 0
 fi
 
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 echo "starting docker registry ..."
-systemctl daemon-reload
+$ssh_cmd systemctl daemon-reload
-systemctl enable registry
+$ssh_cmd systemctl enable registry
-systemctl --no-block start registry
+$ssh_cmd systemctl --no-block start registry

magnum/drivers/common/templates/kubernetes/fragments/add-proxy.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
+set +x
 . /etc/sysconfig/heat-params
+set -x
+
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
 
 DOCKER_HTTP_PROXY_CONF=/etc/systemd/system/docker.service.d/http_proxy.conf
 
@@ -60,6 +64,6 @@ EOF
 fi
 
 if [ "$DOCKER_RESTART" -eq 1 ]; then
-    systemctl daemon-reload
+    $ssh_cmd systemctl daemon-reload
-    systemctl --no-block restart docker.service
+    $ssh_cmd systemctl --no-block restart docker.service
 fi

magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh

@@ -4,6 +4,8 @@
 
 set -x
 
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 if [ ! -z "$HTTP_PROXY" ]; then
     export HTTP_PROXY
 fi
@@ -20,13 +22,13 @@ if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
 
     attempts=60
     while [ ${attempts} -gt 0 ]; do
-        device_name=$(ls /dev/disk/by-id | grep ${ETCD_VOLUME:0:20}$)
+        device_name=$($ssh_cmd ls /dev/disk/by-id | grep ${ETCD_VOLUME:0:20}$)
         if [ -n "${device_name}" ]; then
             break
         fi
         echo "waiting for disk device"
         sleep 0.5
-        udevadm trigger
+        $ssh_cmd udevadm trigger
         let attempts--
     done
 
@@ -36,20 +38,20 @@ if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
     fi
 
     device_path=/dev/disk/by-id/${device_name}
-    fstype=$(blkid -s TYPE -o value ${device_path})
+    fstype=$($ssh_cmd blkid -s TYPE -o value ${device_path} || echo "")
     if [ "${fstype}" != "xfs" ]; then
-        mkfs.xfs -f ${device_path}
+        $ssh_cmd mkfs.xfs -f ${device_path}
     fi
-    mkdir -p /var/lib/etcd
+    $ssh_cmd mkdir -p /var/lib/etcd
     echo "${device_path} /var/lib/etcd xfs defaults 0 0" >> /etc/fstab
-    mount -a
+    $ssh_cmd mount -a
-    chown -R etcd.etcd /var/lib/etcd
+    $ssh_cmd chown -R etcd.etcd /var/lib/etcd
-    chmod 755 /var/lib/etcd
+    $ssh_cmd chmod 755 /var/lib/etcd
 
 fi
 
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
-atomic install \
+$ssh_cmd atomic install \
 --system-package no \
 --system \
 --storage ostree \

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -4,6 +4,8 @@
 
 echo "configuring kubernetes (master)"
 
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 if [ ! -z "$HTTP_PROXY" ]; then
     export HTTP_PROXY
 fi
@@ -18,7 +20,7 @@ fi
 
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
 
-mkdir -p /opt/cni
+$ssh_cmd mkdir -p /opt/cni
 _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
 
 if [ "$NETWORK_DRIVER" = "calico" ]; then
@@ -36,11 +38,19 @@ EOF
     fi
 fi
 
-atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
+
+mkdir -p /srv/magnum/kubernetes/
+cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
+#!/bin/bash -x
+atomic install --storage ostree --system --set=ADDTL_MOUNTS='${_addtl_mounts}' --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
+EOF
+chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
+$ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"
+
 
 CERT_DIR=/etc/kubernetes/certs
 
@@ -177,8 +187,8 @@ sed -i '
 
 sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
 
-mkdir -p /etc/kubernetes/manifests
+$ssh_cmd mkdir -p /etc/kubernetes/manifests
-HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
+HOSTNAME_OVERRIDE=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
 KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${HOSTNAME_OVERRIDE}"
 KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
 KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
@@ -201,7 +211,7 @@ KUBELET_ARGS="${KUBELET_ARGS} --register-with-taints=CriticalAddonsOnly=True:NoS
 KUBELET_ARGS="${KUBELET_ARGS} --node-labels=node-role.kubernetes.io/master=\"\""
 
 KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
-HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
+HOSTNAME_OVERRIDE=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
 cat << EOF >> ${KUBELET_KUBECONFIG}
 apiVersion: v1
 clusters:
@@ -241,9 +251,9 @@ KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-fil
 # specified cgroup driver
 KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=${CGROUP_DRIVER}"
 
-systemctl disable docker
+$ssh_cmd systemctl disable docker
-if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
+if $ssh_cmd cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
-        cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
+        $ssh_cmd cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
         sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
                 /etc/systemd/system/docker.service
 else
@@ -253,8 +263,8 @@ EOF
 
 fi
 
-systemctl daemon-reload
+$ssh_cmd systemctl daemon-reload
-systemctl enable docker
+$ssh_cmd systemctl enable docker
 
 if [ -z "${KUBE_NODE_IP}" ]; then
     KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
@@ -267,4 +277,3 @@ sed -i '
 /^KUBELET_HOSTNAME=/ s/=.*/=""/
 /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
 ' /etc/kubernetes/kubelet
-

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -1,6 +1,10 @@
-#!/bin/sh -x
+#!/bin/sh
 
+set +x
 . /etc/sysconfig/heat-params
+set -x
+
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
 
 echo "configuring kubernetes (minion)"
 
@@ -19,11 +23,11 @@ fi
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
 
 _addtl_mounts=''
-mkdir -p /opt/cni
+$ssh_cmd mkdir -p /opt/cni
 _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
 
 if [ "$NETWORK_DRIVER" = "calico" ]; then
-    if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then
+    if [ "$($ssh_cmd systemctl status NetworkManager.service | grep -o "Active: active")" = "Active: active" ]; then
         CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
         [ -f ${CALICO_NM} ] || {
         echo "Writing File: $CALICO_NM"
@@ -33,22 +37,26 @@ if [ "$NETWORK_DRIVER" = "calico" ]; then
 unmanaged-devices=interface-name:cali*;interface-name:tunl*
 EOF
 }
-        systemctl restart NetworkManager
+        $ssh_cmd systemctl restart NetworkManager
     fi
 fi
 
-atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS=${_addtl_mounts} --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
+mkdir -p /srv/magnum/kubernetes/
+cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
+#!/bin/bash -x
+atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS='${_addtl_mounts}' --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
+EOF
+chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
+$ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"
 
 CERT_DIR=/etc/kubernetes/certs
-PROTOCOL=https
 ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
 KUBE_PROTOCOL="https"
 KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
 PROXY_KUBECONFIG=/etc/kubernetes/proxy-config.yaml
 
 if [ "$TLS_DISABLED" = "True" ]; then
-    PROTOCOL=http
     KUBE_PROTOCOL="http"
 fi
 
@@ -57,7 +65,7 @@ KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
 if [ -z "${KUBE_NODE_IP}" ]; then
     KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 fi
-HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
+HOSTNAME_OVERRIDE="$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')"
 cat << EOF >> ${KUBELET_KUBECONFIG}
 apiVersion: v1
 clusters:
@@ -110,8 +118,8 @@ if [ "$TLS_DISABLED" = "True" ]; then
     sed -i 's/^.*certificate-authority.*$//' ${KUBELET_KUBECONFIG}
 fi
 
-chmod 0644 ${KUBELET_KUBECONFIG}
+chmod 0640 ${KUBELET_KUBECONFIG}
-chmod 0644 ${PROXY_KUBECONFIG}
+chmod 0640 ${PROXY_KUBECONFIG}
 
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
@@ -136,11 +144,6 @@ if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true"
     KUBELET_ARGS="${KUBELET_ARGS} --cloud-provider=external"
 fi
 
-# Workaround for Cinder support (fixed in k8s >= 1.6)
-if [ ! -f /usr/bin/udevadm ]; then
-    ln -s /sbin/udevadm /usr/bin/udevadm
-fi
-
 # For using default log-driver, other options should be ignored
 sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
 
@@ -158,9 +161,9 @@ if [ "$(echo $AUTO_HEALING_ENABLED | tr '[:upper:]' '[:lower:]')" = "true" ]; th
     KUBELET_ARGS="${KUBELET_ARGS} --node-labels=draino-enabled=true"
 fi
 
-systemctl disable docker
+$ssh_cmd systemctl disable docker
-if cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
+if $ssh_cmd cat /usr/lib/systemd/system/docker.service | grep 'native.cgroupdriver'; then
-        cp /usr/lib/systemd/system/docker.service /etc/systemd/system/
+        $ssh_cmd "cp /usr/lib/systemd/system/docker.service /etc/systemd/system/"
         sed -i "s/\(native.cgroupdriver=\)\w\+/\1$CGROUP_DRIVER/" \
                 /etc/systemd/system/docker.service
 else
@@ -170,8 +173,8 @@ EOF
 
 fi
 
-systemctl daemon-reload
+$ssh_cmd systemctl daemon-reload
-systemctl enable docker
+$ssh_cmd systemctl enable docker
 
 cat > /etc/kubernetes/get_require_kubeconfig.sh <<EOF
 #!/bin/bash
@@ -201,4 +204,4 @@ cat >> /etc/environment <<EOF
 KUBERNETES_MASTER=$KUBE_MASTER_URI
 EOF
 
-hostname `hostname | sed 's/.novalocal//'`
+$ssh_cmd "hostname $(cat /etc/hostname | head -1 |sed 's/.novalocal//')"

magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh

@@ -2,8 +2,10 @@
 
 . /etc/sysconfig/heat-params
 
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 # make sure we pick up any modified unit files
-systemctl daemon-reload
+$ssh_cmd systemctl daemon-reload
 
 # if the certificate manager api is enabled, wait for the ca key to be handled
 # by the heat container agent (required for the controller-manager)
@@ -16,6 +18,6 @@ done
 echo "starting services"
 for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy; do
     echo "activating service $service"
-    systemctl enable $service
+    $ssh_cmd systemctl enable $service
-    systemctl --no-block start $service
+    $ssh_cmd systemctl --no-block restart $service
 done

magnum/drivers/common/templates/kubernetes/fragments/enable-services-minion.sh

@@ -1,17 +1,21 @@
 #!/bin/sh
 
+set -x
+
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 # docker is already enabled and possibly running on centos atomic host
 # so we need to stop it first and delete the docker0 bridge (which will
 # be re-created using the flannel-provided subnet).
 echo "stopping docker"
-systemctl stop docker
+$ssh_cmd systemctl stop docker
-ip link del docker0
+$ssh_cmd ip link del docker0
 
 # make sure we pick up any modified unit files
-systemctl daemon-reload
+$ssh_cmd systemctl daemon-reload
 
 for service in docker kubelet kube-proxy; do
     echo "activating service $service"
-    systemctl enable $service
+    $ssh_cmd systemctl enable $service
-    systemctl --no-block start $service
+    $ssh_cmd systemctl --no-block start $service
 done

magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh

@@ -327,4 +327,51 @@ EOF
     kubectl apply -f ${OCCM}
 fi
 
+# Assgin read daemonset/replicaset/statefulset permssion to allow node drain itself
+cat <<EOF | kubectl apply --validate=false -f -
+---
+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: system:node-drainer
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - pods/eviction
+    verbs:
+    - create
+  - apiGroups:
+    - apps
+    resources:
+    - statefulsets
+    verbs:
+    - get
+    - list
+  - apiGroups:
+    - extensions
+    resources:
+    - daemonsets
+    - replicasets
+    verbs:
+    - get
+    - list
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:node-drainer
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:node-drainer
+  subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: Group
+    name: system:nodes
+kind: List
+metadata: {}
+EOF
+
 printf "Finished running ${step}\n"

magnum/drivers/common/templates/kubernetes/fragments/make-cert-client.sh

@@ -14,12 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set +x
 . /etc/sysconfig/heat-params
+set -x
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
+
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 if [ "$TLS_DISABLED" == "True" ]; then
     exit 0
 fi
@@ -77,9 +82,9 @@ EOF
         $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > $CA_CERT
 
     # Generate client's private key and csr
-    openssl genrsa -out "${_KEY}" 4096
+    $ssh_cmd openssl genrsa -out "${_KEY}" 4096
     chmod 400 "${_KEY}"
-    openssl req -new -days 1000 \
+    $ssh_cmd openssl req -new -days 1000 \
             -key "${_KEY}" \
             -out "${_CSR}" \
             -reqexts req_ext \
@@ -96,8 +101,8 @@ EOF
 }
 
 #Kubelet Certs
-INSTANCE_NAME=$(hostname --short | sed 's/\.novalocal//')
+INSTANCE_NAME=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
-HOSTNAME=$(hostname)
+HOSTNAME=$(cat /etc/hostname | head -1)
 
 cat > ${cert_dir}/kubelet.conf <<EOF
 [req]
@@ -138,12 +143,6 @@ EOF
 generate_certificates kubelet ${cert_dir}/kubelet.conf
 generate_certificates proxy ${cert_dir}/proxy.conf
 
-# Common certs and key are created for both etcd and kubernetes services.
-# Both etcd and kube user should have permission to access the certs and key.
-groupadd kube_etcd
-usermod -a -G kube_etcd etcd
-usermod -a -G kube_etcd kube
 chmod 550 "${cert_dir}"
-chown -R kube:kube_etcd "${cert_dir}"
+chmod 440 "${cert_dir}/kubelet.key"
-chmod 440 ${cert_dir}/kubelet.key
+chmod 440 "${cert_dir}/proxy.key"
-chmod 440 ${cert_dir}/proxy.key

magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh

@@ -21,6 +21,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+
 if [ "$TLS_DISABLED" == "True" ]; then
     exit 0
 fi
@@ -115,9 +117,9 @@ EOF
         $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
 
     # Generate server's private key and csr
-    openssl genrsa -out "${_KEY}" 4096
+    $ssh_cmd openssl genrsa -out "${_KEY}" 4096
     chmod 400 "${_KEY}"
-    openssl req -new -days 1000 \
+    $ssh_cmd openssl req -new -days 1000 \
             -key "${_KEY}" \
             -out "${_CSR}" \
             -reqexts req_ext \
@@ -147,7 +149,7 @@ extendedKeyUsage = clientAuth,serverAuth
 EOF
 
 #Kubelet Certs
-INSTANCE_NAME=$(hostname --short | sed 's/\.novalocal//')
+INSTANCE_NAME=$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')
 cat > ${cert_dir}/kubelet.conf <<EOF
 [req]
 distinguished_name = req_distinguished_name
@@ -175,11 +177,11 @@ echo -e "${KUBE_SERVICE_ACCOUNT_PRIVATE_KEY}" > ${cert_dir}/service_account_priv
 
 # Common certs and key are created for both etcd and kubernetes services.
 # Both etcd and kube user should have permission to access the certs and key.
-groupadd kube_etcd
+$ssh_cmd groupadd kube_etcd
-usermod -a -G kube_etcd etcd
+$ssh_cmd usermod -a -G kube_etcd etcd
-usermod -a -G kube_etcd kube
+$ssh_cmd usermod -a -G kube_etcd kube
-chmod 550 "${cert_dir}"
+$ssh_cmd chmod 550 "${cert_dir}"
-chown -R kube:kube_etcd "${cert_dir}"
+$ssh_cmd chown -R kube:kube_etcd "${cert_dir}"
-chmod 440 $cert_dir/server.key
+$ssh_cmd chmod 440 "$cert_dir/server.key"
-mkdir -p /etc/etcd/certs
+$ssh_cmd mkdir -p /etc/etcd/certs
-cp ${cert_dir}/* /etc/etcd/certs
+$ssh_cmd cp ${cert_dir}/* /etc/etcd/certs

magnum/drivers/common/templates/kubernetes/fragments/start-container-agent.sh

@@ -1,18 +1,23 @@
 #!/bin/bash
 
-. /etc/sysconfig/heat-params
+set -x
+set +u
+HTTP_PROXY="$HTTP_PROXY"
+HTTPS_PROXY="$HTTPS_PROXY"
+NO_PROXY="$NO_PROXY"
+CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX"
+HEAT_CONTAINER_AGENT_TAG="$HEAT_CONTAINER_AGENT_TAG"
 
-set -uxe
 
-if [ ! -z "$HTTP_PROXY" ]; then
+if [ -n "${HTTP_PROXY}" ]; then
     export HTTP_PROXY
 fi
 
-if [ ! -z "$HTTPS_PROXY" ]; then
+if [ -n "${HTTPS_PROXY}" ]; then
     export HTTPS_PROXY
 fi
 
-if [ ! -z "$NO_PROXY" ]; then
+if [ -n "${NO_PROXY}" ]; then
     export NO_PROXY
 fi
 
@@ -21,7 +26,7 @@ fi
 # in host mount namespace and apply configuration.
 mkdir -p /srv/magnum/.ssh
 chmod 700 /srv/magnum/.ssh
-ssh-keygen -t rsa -N '' -f /srv/magnum/.ssh/heat_agent_rsa
+ssh-keygen -q -t rsa -N '' -f /srv/magnum/.ssh/heat_agent_rsa
 chmod 400 /srv/magnum/.ssh/heat_agent_rsa
 chmod 400 /srv/magnum/.ssh/heat_agent_rsa.pub
 # Add the public to the host authorized_keys file.
@@ -41,13 +46,13 @@ sed -i '/^PermitRootLogin/ s/ .*/ without-password/' /etc/ssh/sshd_config
 systemctl restart sshd
 
 
-_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
+_prefix="${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}"
 atomic install \
 --storage ostree \
 --system \
 --system-package no \
 --set REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt \
 --name heat-container-agent \
-${_prefix}heat-container-agent:${HEAT_CONTAINER_AGENT_TAG}
+"${_prefix}heat-container-agent:${HEAT_CONTAINER_AGENT_TAG}"
 
 systemctl start heat-container-agent

magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. /etc/sysconfig/heat-params
+set -x
+
+ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
+kubecontrol="/var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml"
+new_kube_tag="$kube_tag_input"
+
+if [ ${new_kube_tag}!=${KUBE_TAG} ]; then
+    HOSTNAME_OVERRIDE="$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')"
+    # If there is only one master and this is the master node, skip the drain, just cordon it
+    # If there is only one worker and this is the worker node, skip the drain, just cordon it
+    all_masters=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master= -o name)
+    all_workers=$(${ssh_cmd} ${kubecontrol} get nodes --selector=node-role.kubernetes.io/master!= -o name)
+    if [ "node/${HOSTNAME_OVERRIDE}" != "${all_masters}" ] && [ "node/${HOSTNAME_OVERRIDE}" != "${all_workers}" ]; then
+        ${ssh_cmd} ${kubecontrol} drain ${HOSTNAME_OVERRIDE} --ignore-daemonsets --delete-local-data --force
+    else
+        ${ssh_cmd} ${kubecontrol} cordon ${HOSTNAME_OVERRIDE}
+    fi
+
+    declare -A service_image_mapping
+    service_image_mapping=( ["kubelet"]="kubernetes-kubelet" ["kube-controller-manager"]="kubernetes-controller-manager" ["kube-scheduler"]="kubernetes-scheduler" ["kube-proxy"]="kubernetes-proxy" ["kube-apiserver"]="kubernetes-apiserver" )
+
+    SERVICE_LIST=$($ssh_cmd atomic containers list -f container=kube -q --no-trunc)
+
+    for service in ${SERVICE_LIST}; do
+        ${ssh_cmd} systemctl stop ${service}
+    done
+
+    for service in ${SERVICE_LIST}; do
+        ${ssh_cmd} atomic pull --storage ostree "docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag}"
+    done
+
+    for service in ${SERVICE_LIST}; do
+        ${ssh_cmd} atomic containers update --rebase docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag} ${service}
+    done
+
+    for service in ${SERVICE_LIST}; do
+        systemctl restart ${service}
+    done
+
+    ${ssh_cmd} /var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml uncordon ${HOSTNAME_OVERRIDE}
+
+    # FIXME(flwang): The KUBE_TAG could be out of date after a successful upgrade
+    for service in ${SERVICE_LIST}; do
+        ${ssh_cmd} atomic --assumeyes images "delete docker.io/openstackmagnum/${service_image_mapping[${service}]}:${KUBE_TAG}"
+    done
+
+    ${ssh_cmd} atomic images prune
+
+fi

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh

@@ -1,10 +1,12 @@
-#cloud-config
+#!/bin/sh
-merge_how: dict(recurse_array)+list(append)
+
-write_files:
+echo "START: write-heat-params"
-  - path: /etc/sysconfig/heat-params
+
-    owner: "root:root"
+HEAT_PARAMS=/etc/sysconfig/heat-params
-    permissions: "0600"
+[ -f ${HEAT_PARAMS} ] || {
-    content: |
+    echo "Writing File: $HEAT_PARAMS"
+    mkdir -p "$(dirname ${HEAT_PARAMS})"
+    cat > ${HEAT_PARAMS} <<EOF
       PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING"
       KUBE_API_PUBLIC_ADDRESS="$KUBE_API_PUBLIC_ADDRESS"
       KUBE_API_PRIVATE_ADDRESS="$KUBE_API_PRIVATE_ADDRESS"
@@ -98,3 +100,10 @@ write_files:
       AUTOSCALER_TAG="$AUTOSCALER_TAG"
       MIN_NODE_COUNT="$MIN_NODE_COUNT"
       MAX_NODE_COUNT="$MAX_NODE_COUNT"
+EOF
+}
+
+chown root:root "${HEAT_PARAMS}"
+chmod 600 "${HEAT_PARAMS}"
+
+echo "END: write-heat-params"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+
+echo "START: write-heat-params"
+
+HEAT_PARAMS=/etc/sysconfig/heat-params
+[ -f ${HEAT_PARAMS} ] || {
+    echo "Writing File: $HEAT_PARAMS"
+    mkdir -p "$(dirname ${HEAT_PARAMS})"
+    cat > ${HEAT_PARAMS} <<EOF
+PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING"
+KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
+KUBE_MASTER_IP="$KUBE_MASTER_IP"
+KUBE_API_PORT="$KUBE_API_PORT"
+KUBE_NODE_PUBLIC_IP="$KUBE_NODE_PUBLIC_IP"
+KUBE_NODE_IP="$KUBE_NODE_IP"
+ETCD_SERVER_IP="$ETCD_SERVER_IP"
+ENABLE_CINDER="$ENABLE_CINDER"
+DOCKER_VOLUME="$DOCKER_VOLUME"
+DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
+DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
+CGROUP_DRIVER="$CGROUP_DRIVER"
+NETWORK_DRIVER="$NETWORK_DRIVER"
+REGISTRY_ENABLED="$REGISTRY_ENABLED"
+REGISTRY_PORT="$REGISTRY_PORT"
+SWIFT_REGION="$SWIFT_REGION"
+REGISTRY_CONTAINER="$REGISTRY_CONTAINER"
+REGISTRY_INSECURE="$REGISTRY_INSECURE"
+REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE"
+TLS_DISABLED="$TLS_DISABLED"
+VERIFY_CA="$VERIFY_CA"
+CLUSTER_UUID="$CLUSTER_UUID"
+MAGNUM_URL="$MAGNUM_URL"
+AUTH_URL="$AUTH_URL"
+USERNAME="$USERNAME"
+PASSWORD="$PASSWORD"
+VOLUME_DRIVER="$VOLUME_DRIVER"
+REGION_NAME="$REGION_NAME"
+HTTP_PROXY="$HTTP_PROXY"
+HTTPS_PROXY="$HTTPS_PROXY"
+NO_PROXY="$NO_PROXY"
+WAIT_CURL="$WAIT_CURL"
+KUBE_TAG="$KUBE_TAG"
+FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
+PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR"
+KUBE_VERSION="$KUBE_VERSION"
+TRUSTEE_USER_ID="$TRUSTEE_USER_ID"
+TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD"
+TRUST_ID="$TRUST_ID"
+CLOUD_PROVIDER_ENABLED="$CLOUD_PROVIDER_ENABLED"
+INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL"
+CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX"
+DNS_SERVICE_IP="$DNS_SERVICE_IP"
+DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
+KUBELET_OPTIONS="$KUBELET_OPTIONS"
+KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"
+OCTAVIA_ENABLED="$OCTAVIA_ENABLED"
+HEAT_CONTAINER_AGENT_TAG="$HEAT_CONTAINER_AGENT_TAG"
+EOF
+}
+
+chown root:root "${HEAT_PARAMS}"
+chmod 600 "${HEAT_PARAMS}"
+
+echo "END: write-heat-params"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml

@@ -1,59 +0,0 @@
-#cloud-config
-merge_how: dict(recurse_array)+list(append)
-write_files:
-  - path: /etc/sysconfig/heat-params
-    owner: "root:root"
-    permissions: "0600"
-    content: |
-      PROMETHEUS_MONITORING="$PROMETHEUS_MONITORING"
-      KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
-      KUBE_MASTER_IP="$KUBE_MASTER_IP"
-      KUBE_API_PORT="$KUBE_API_PORT"
-      KUBE_NODE_PUBLIC_IP="$KUBE_NODE_PUBLIC_IP"
-      KUBE_NODE_IP="$KUBE_NODE_IP"
-      ETCD_SERVER_IP="$ETCD_SERVER_IP"
-      ENABLE_CINDER="$ENABLE_CINDER"
-      DOCKER_VOLUME="$DOCKER_VOLUME"
-      DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
-      DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
-      CGROUP_DRIVER="$CGROUP_DRIVER"
-      NETWORK_DRIVER="$NETWORK_DRIVER"
-      REGISTRY_ENABLED="$REGISTRY_ENABLED"
-      REGISTRY_PORT="$REGISTRY_PORT"
-      SWIFT_REGION="$SWIFT_REGION"
-      REGISTRY_CONTAINER="$REGISTRY_CONTAINER"
-      REGISTRY_INSECURE="$REGISTRY_INSECURE"
-      REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE"
-      TLS_DISABLED="$TLS_DISABLED"
-      TRAEFIK_INGRESS_CONTROLLER_TAG="$TRAEFIK_INGRESS_CONTROLLER_TAG"
-      VERIFY_CA="$VERIFY_CA"
-      CLUSTER_UUID="$CLUSTER_UUID"
-      MAGNUM_URL="$MAGNUM_URL"
-      AUTH_URL="$AUTH_URL"
-      USERNAME="$USERNAME"
-      PASSWORD="$PASSWORD"
-      VOLUME_DRIVER="$VOLUME_DRIVER"
-      REGION_NAME="$REGION_NAME"
-      HTTP_PROXY="$HTTP_PROXY"
-      HTTPS_PROXY="$HTTPS_PROXY"
-      NO_PROXY="$NO_PROXY"
-      WAIT_CURL="$WAIT_CURL"
-      KUBE_TAG="$KUBE_TAG"
-      FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
-      PODS_NETWORK_CIDR="$PODS_NETWORK_CIDR"
-      KUBE_VERSION="$KUBE_VERSION"
-      TRUSTEE_USER_ID="$TRUSTEE_USER_ID"
-      TRUSTEE_USERNAME="$TRUSTEE_USERNAME"
-      TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD"
-      TRUSTEE_DOMAIN_ID="$TRUSTEE_DOMAIN_ID"
-      TRUST_ID="$TRUST_ID"
-      CLOUD_PROVIDER_ENABLED="$CLOUD_PROVIDER_ENABLED"
-      INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL"
-      CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX"
-      DNS_SERVICE_IP="$DNS_SERVICE_IP"
-      DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
-      KUBELET_OPTIONS="$KUBELET_OPTIONS"
-      KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"
-      OCTAVIA_ENABLED="$OCTAVIA_ENABLED"
-      HEAT_CONTAINER_AGENT_TAG="$HEAT_CONTAINER_AGENT_TAG"
-      AUTO_HEALING_ENABLED="$AUTO_HEALING_ENABLED"

magnum/drivers/common/templates/kubernetes/fragments/write-kube-os-config.sh

@@ -1,15 +1,17 @@
 #!/bin/sh
 
+set +x
 . /etc/sysconfig/heat-params
+set -x
 
-mkdir -p /etc/kubernetes/
+$ssh_cmd mkdir -p /etc/kubernetes/
 
 if [ -z "${TRUST_ID}" ]; then
     exit 0
 fi
 
 KUBE_OS_CLOUD_CONFIG=/etc/kubernetes/cloud-config
-cp /etc/pki/tls/certs/ca-bundle.crt /etc/kubernetes/ca-bundle.crt
+$ssh_cmd cp /etc/pki/tls/certs/ca-bundle.crt /etc/kubernetes/ca-bundle.crt
 
 # Generate a the configuration for Kubernetes services
 # to talk to OpenStack Neutron and Cinder
@@ -33,9 +35,9 @@ bs-version=v2
 EOF
 
 # Provide optional region parameter if it's set.
-if [ -n ${REGION_NAME} ]; then
+if [ -n "${REGION_NAME}" ]; then
     sed -i '/ca-file/a region='${REGION_NAME}'' $KUBE_OS_CLOUD_CONFIG
 fi
 
 # backwards compatibility, some apps may expect this file from previous magnum versions.
-cp ${KUBE_OS_CLOUD_CONFIG} /etc/kubernetes/kube_openstack_config
+$ssh_cmd cp ${KUBE_OS_CLOUD_CONFIG} /etc/kubernetes/kube_openstack_config

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -115,6 +115,9 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                 '"cinder" volume driver needs "cloud_provider_enabled" label '
                 'to be true or unset.'))
 
+        extra_params['master_image'] = cluster_template.image_id
+        extra_params['minion_image'] = cluster_template.image_id
+
         label_list = ['coredns_tag',
                       'kube_tag', 'container_infra_prefix',
                       'availability_zone',

magnum/drivers/heat/template_def.py

@@ -472,6 +472,9 @@ class BaseTemplateDefinition(TemplateDefinition):
 
     def get_discovery_url(self, cluster, cluster_template=None):
         if hasattr(cluster, 'discovery_url') and cluster.discovery_url:
+            # NOTE(flwang): The discovery URl does have a expiry time,
+            # so better skip it when the cluster has been created.
+            if not cluster.master_addresses:
                 self.validate_discovery_url(cluster.discovery_url,
                                             cluster.master_count)
             discovery_url = cluster.discovery_url

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -30,6 +30,14 @@ parameters:
     type: string
     description: glance image used to boot the server
 
+  master_image:
+    type: string
+    description: glance image used to boot the server
+
+  minion_image:
+    type: string
+    description: glance image used to boot the server
+
   master_flavor:
     type: string
     default: m1.small
@@ -332,6 +340,16 @@ parameters:
     description: tag of the k8s containers used to provision the kubernetes cluster
     default: v1.11.6
 
+  master_kube_tag:
+    type: string
+    description: tag of the k8s containers used to provision the kubernetes cluster
+    default: v1.11.6
+
+  minion_kube_tag:
+    type: string
+    description: tag of the k8s containers used to provision the kubernetes cluster
+    default: v1.11.6
+
   # FIXME update cloud_provider_tag when a fix for PVC is released
   # https://github.com/kubernetes/cloud-provider-openstack/pull/405
   cloud_provider_tag:
@@ -620,6 +638,13 @@ parameters:
     description: >
       maximum node count of cluster workers when doing scale up
 
+  update_max_batch_size:
+    type: number
+    description: >
+      max batch size when doing rolling upgrade
+    default: 1
+
+
 resources:
 
   ######################################################################
@@ -707,6 +732,9 @@ resources:
         # well, we could remove this rule here.
         # The PR in ccm is
         # https://github.com/kubernetes/cloud-provider-openstack/pull/491
+        - protocol: tcp
+          port_range_min: 22
+          port_range_max: 22
         - protocol: tcp
           port_range_min: 30000
           port_range_max: 32767
@@ -798,6 +826,8 @@ resources:
     type: OS::Heat::ResourceGroup
     depends_on:
       - network
+    update_policy:
+      rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
     properties:
       count: {get_param: number_of_masters}
       resource_def:
@@ -812,7 +842,7 @@ resources:
           api_public_address: {get_attr: [api_lb, floating_address]}
           api_private_address: {get_attr: [api_lb, address]}
           ssh_key_name: {get_param: ssh_key_name}
-          server_image: {get_param: server_image}
+          server_image: {get_param: master_image}
           master_flavor: {get_param: master_flavor}
           external_network: {get_param: external_network}
           kube_allow_priv: {get_param: kube_allow_priv}
@@ -850,7 +880,7 @@ resources:
           http_proxy: {get_param: http_proxy}
           https_proxy: {get_param: https_proxy}
           no_proxy: {get_param: no_proxy}
-          kube_tag: {get_param: kube_tag}
+          kube_tag: {get_param: master_kube_tag}
           cloud_provider_tag: {get_param: cloud_provider_tag}
           cloud_provider_enabled: {get_param: cloud_provider_enabled}
           kube_version: {get_param: kube_version}
@@ -968,6 +998,8 @@ resources:
     type: OS::Heat::ResourceGroup
     depends_on:
       - network
+    update_policy:
+      rolling_update: {max_batch_size: {get_param: update_max_batch_size}, pause_time: 30}
     properties:
       count: {get_param: number_of_minions}
       removal_policies: [{resource_list: {get_param: minions_to_remove}}]
@@ -980,7 +1012,7 @@ resources:
               - [{ get_param: 'OS::stack_name' }, 'minion', '%index%']
           prometheus_monitoring: {get_param: prometheus_monitoring}
           ssh_key_name: {get_param: ssh_key_name}
-          server_image: {get_param: server_image}
+          server_image: {get_param: minion_image}
           minion_flavor: {get_param: minion_flavor}
           fixed_network: {get_attr: [network, fixed_network]}
           fixed_subnet: {get_attr: [network, fixed_subnet]}
@@ -1015,7 +1047,7 @@ resources:
           http_proxy: {get_param: http_proxy}
           https_proxy: {get_param: https_proxy}
           no_proxy: {get_param: no_proxy}
-          kube_tag: {get_param: kube_tag}
+          kube_tag: {get_param: minion_kube_tag}
           kube_version: {get_param: kube_version}
           trustee_user_id: {get_param: trustee_user_id}
           trustee_username: {get_param: trustee_username}

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -504,13 +504,35 @@ resources:
   # a multipart MIME user-data archive.
   #
 
-  write_heat_params:
+  agent_config:
     type: OS::Heat::SoftwareConfig
     properties:
       group: ungrouped
       config:
-        str_replace:
+        list_join:
-          template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.yaml}
+          - "\n"
+          -
+            - str_replace:
+                template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh}
+                params:
+                  $OPENSTACK_CA: {get_param: openstack_ca}
+            - str_replace:
+                template: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
+                params:
+                  $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
+                  $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
+            - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh
+
+  master_config:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+          - "\n"
+          -
+            - str_replace:
+                template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params-master.sh}
                 params:
                   "$PROMETHEUS_MONITORING": {get_param: prometheus_monitoring}
                   "$KUBE_API_PUBLIC_ADDRESS": {get_attr: [api_address_switch, public_ip]}
@@ -604,90 +626,24 @@ resources:
                   "$AUTOSCALER_TAG": {get_param: autoscaler_tag}
                   "$MIN_NODE_COUNT": {get_param: min_node_count}
                   "$MAX_NODE_COUNT": {get_param: max_node_count}
-
+            - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
-  install_openstack_ca:
+            - get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh
-    type: OS::Heat::SoftwareConfig
+            - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
-    properties:
+            - get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-master.sh
-      group: ungrouped
+            - str_replace:
-      config:
+                template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh}
-        str_replace:
-          params:
-            $OPENSTACK_CA: {get_param: openstack_ca}
-          template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh}
-
-  make_cert:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/make-cert.sh}
-
-  configure_docker_storage:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config:
-        str_replace:
                 params:
                   $configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh}
-          template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh}
+            - get_file: ../../common/templates/kubernetes/fragments/enable-services-master.sh
+            - get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh
 
-  configure_etcd:
+  master_config_deployment:
-    type: OS::Heat::SoftwareConfig
+    type: OS::Heat::SoftwareDeployment
     properties:
-      group: ungrouped
+      signal_transport: HEAT_SIGNAL
-      config: {get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh}
+      config: {get_resource: master_config}
-
+      server: {get_resource: kube-master}
-  write_kube_os_config:
+      actions: ['CREATE']
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh}
-
-  configure_kubernetes:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-master.sh}
-
-  enable_services:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/enable-services-master.sh}
-
-  disable_selinux:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh}
-
-  add_proxy:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh}
-
-  start_container_agent:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
-
-  kube_master_init:
-    type: OS::Heat::MultipartMime
-    properties:
-      parts:
-        - config: {get_resource: install_openstack_ca}
-        - config: {get_resource: disable_selinux}
-        - config: {get_resource: write_heat_params}
-        - config: {get_resource: configure_etcd}
-        - config: {get_resource: write_kube_os_config}
-        - config: {get_resource: configure_docker_storage}
-        - config: {get_resource: configure_kubernetes}
-        - config: {get_resource: make_cert}
-        - config: {get_resource: add_proxy}
-        - config: {get_resource: start_container_agent}
-        - config: {get_resource: enable_services}
 
   ######################################################################
   #
@@ -706,7 +662,7 @@ resources:
       key_name: {get_param: ssh_key_name}
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
-      user_data: {get_resource: kube_master_init}
+      user_data: {get_resource: agent_config}
       networks:
         - port: {get_resource: kube_master_eth0}
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
@@ -783,6 +739,25 @@ resources:
       volume_id: {get_resource: docker_volume}
       mountpoint: /dev/vdb
 
+  upgrade_kubernetes:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+      - name: kube_tag_input
+      config:
+        get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
+
+  upgrade_kubernetes_deployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      signal_transport: HEAT_SIGNAL
+      config: {get_resource: upgrade_kubernetes}
+      server: {get_resource: kube-master}
+      actions: ['UPDATE']
+      input_values:
+        kube_tag_input: {get_param: kube_tag}
+
 outputs:
 
   OS::stack_id:

magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml

@@ -283,21 +283,24 @@ parameters:
 
 resources:
 
-  start_container_agent:
+  agent_config:
     type: OS::Heat::SoftwareConfig
     properties:
       group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
+      config:
-
+        list_join:
-  minion_wait_handle:
+          - "\n"
-    type: OS::Heat::WaitConditionHandle
+          -
-
+            - str_replace:
-  minion_wait_condition:
+                template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh}
-    type: OS::Heat::WaitCondition
+                params:
-    depends_on: kube-minion
+                  $OPENSTACK_CA: {get_param: openstack_ca}
-    properties:
+            - str_replace:
-      handle: {get_resource: minion_wait_handle}
+                template: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
-      timeout: {get_param: wait_condition_timeout}
+                params:
+                  $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
+                  $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
+            - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh
 
   ######################################################################
   #
@@ -305,13 +308,16 @@ resources:
   # a multipart MIME user-data archive.
   #
 
-  write_heat_params:
+  node_config:
     type: OS::Heat::SoftwareConfig
     properties:
-      group: ungrouped
+      group: script
       config:
-        str_replace:
+        list_join:
-          template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.yaml}
+          - "\n"
+          -
+            - str_replace:
+                template: {get_file: ../../common/templates/kubernetes/fragments/write-heat-params.sh}
                 params:
                   $PROMETHEUS_MONITORING: {get_param: prometheus_monitoring}
                   $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
@@ -346,11 +352,8 @@ resources:
                   $FLANNEL_NETWORK_CIDR: {get_param: flannel_network_cidr}
                   $PODS_NETWORK_CIDR: {get_param: pods_network_cidr}
                   $KUBE_VERSION: {get_param: kube_version}
-            $WAIT_CURL: {get_attr: [minion_wait_handle, curl_cli]}
                   $TRUSTEE_USER_ID: {get_param: trustee_user_id}
-            $TRUSTEE_USERNAME: {get_param: trustee_username}
                   $TRUSTEE_PASSWORD: {get_param: trustee_password}
-            $TRUSTEE_DOMAIN_ID: {get_param: trustee_domain_id}
                   $TRUST_ID: {get_param: trust_id}
                   $AUTH_URL: {get_param: auth_url}
                   $CLOUD_PROVIDER_ENABLED: {get_param: cloud_provider_enabled}
@@ -363,111 +366,25 @@ resources:
                   $OCTAVIA_ENABLED: {get_param: octavia_enabled}
                   $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
                   $AUTO_HEALING_ENABLED: {get_param: auto_healing_enabled}
-
+            - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
-
+            - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
-  install_openstack_ca:
+            - get_file: ../../common/templates/fragments/configure-docker-registry.sh
-    type: OS::Heat::SoftwareConfig
+            - get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh
-    properties:
+            - get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh
-      group: ungrouped
+            - str_replace:
-      config:
+                template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh}
-        str_replace:
-          params:
-            $OPENSTACK_CA: {get_param: openstack_ca}
-          template: {get_file: ../../common/templates/fragments/atomic-install-openstack-ca.sh}
-
-  write_kube_os_config:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh}
-
-  make_cert:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh}
-
-  configure_docker_storage:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config:
-        str_replace:
                 params:
                   $configure_docker_storage_driver: {get_file: ../../common/templates/fragments/configure_docker_storage_driver_atomic.sh}
-          template: {get_file: ../../common/templates/fragments/configure-docker-storage.sh}
+            - get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh
+            - get_file: ../../common/templates/fragments/enable-docker-registry.sh
 
-  configure_docker_registry:
+  node_config_deployment:
-    type: OS::Heat::SoftwareConfig
+    type: OS::Heat::SoftwareDeployment
     properties:
-      group: ungrouped
+      signal_transport: HEAT_SIGNAL
-      config: {get_file: ../../common/templates/fragments/configure-docker-registry.sh}
+      config: {get_resource: node_config}
-
+      server: {get_resource: kube-minion}
-  configure_kubernetes_minion:
+      actions: ['CREATE']
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/configure-kubernetes-minion.sh}
-
-  enable_services:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/enable-services-minion.sh}
-
-  enable_docker_registry:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/fragments/enable-docker-registry.sh}
-
-  minion_wc_notify:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config:
-        str_replace:
-          template: |
-            #!/bin/bash -v
-            if [ "verify_ca" == "True" ]; then
-                VERIFY_CA=""
-            else
-                VERIFY_CA="-k"
-            fi
-            wc_notify $VERIFY_CA --data-binary '{"status": "SUCCESS"}'
-          params:
-            wc_notify: {get_attr: [minion_wait_handle, curl_cli]}
-            verify_ca: {get_param: verify_ca}
-
-  disable_selinux:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh}
-
-  add_proxy:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: ungrouped
-      config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh}
-
-  kube_minion_init:
-    type: OS::Heat::MultipartMime
-    properties:
-      parts:
-        - config: {get_resource: install_openstack_ca}
-        - config: {get_resource: disable_selinux}
-        - config: {get_resource: write_heat_params}
-        - config: {get_resource: start_container_agent}
-        - config: {get_resource: write_kube_os_config}
-        - config: {get_resource: make_cert}
-        - config: {get_resource: configure_docker_storage}
-        - config: {get_resource: configure_docker_registry}
-        - config: {get_resource: configure_kubernetes_minion}
-        - config: {get_resource: add_proxy}
-        - config: {get_resource: enable_services}
-        - config: {get_resource: enable_docker_registry}
-        - config: {get_resource: minion_wc_notify}
 
   ######################################################################
   #
@@ -484,8 +401,9 @@ resources:
       image: {get_param: server_image}
       flavor: {get_param: minion_flavor}
       key_name: {get_param: ssh_key_name}
-      user_data_format: RAW
+      user_data: {get_resource: agent_config}
-      user_data: {get_resource: kube_minion_init}
+      user_data_format: SOFTWARE_CONFIG
+      software_config_transport: POLL_SERVER_HEAT
       networks:
         - port: {get_resource: kube_minion_eth0}
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
@@ -528,6 +446,25 @@ resources:
       volume_id: {get_resource: docker_volume}
       mountpoint: /dev/vdb
 
+  upgrade_kubernetes:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+      - name: kube_tag_input
+      config:
+        get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
+
+  upgrade_kubernetes_deployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      signal_transport: HEAT_SIGNAL
+      config: {get_resource: upgrade_kubernetes}
+      server: {get_resource: kube-minion}
+      actions: ['UPDATE']
+      input_values:
+        kube_tag_input: {get_param: kube_tag}
+
 outputs:
 
   kube_minion_ip:

magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

@@ -333,6 +333,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             'portal_network_cidr': '10.254.0.0/16',
             'project_id': 'project_id',
             'max_node_count': 2,
+            'master_image': 'image_id',
+            'minion_image': 'image_id',
         }
         if missing_attr is not None:
             expected.pop(mapping[missing_attr], None)
@@ -340,6 +342,10 @@ class TestClusterConductorWithK8s(base.TestCase):
         if missing_attr == 'node_count':
             expected['max_node_count'] = None
 
+        if missing_attr == 'image_id':
+            expected['master_image'] = None
+            expected['minion_image'] = None
+
         self.assertEqual(expected, definition)
         self.assertEqual(
             ['../../common/templates/environments/no_private_network.yaml',
@@ -465,6 +471,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             'portal_network_cidr': '10.254.0.0/16',
             'project_id': 'project_id',
             'max_node_count': 2,
+            'master_image': 'image_id',
+            'minion_image': 'image_id',
         }
 
         self.assertEqual(expected, definition)
@@ -581,6 +589,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             'portal_network_cidr': '10.254.0.0/16',
             'project_id': 'project_id',
             'max_node_count': 2,
+            'master_image': None,
+            'minion_image': None,
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -1008,6 +1018,8 @@ class TestClusterConductorWithK8s(base.TestCase):
             'portal_network_cidr': '10.254.0.0/16',
             'project_id': 'project_id',
             'max_node_count': 2,
+            'master_image': 'image_id',
+            'minion_image': 'image_id',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(

magnum/tests/unit/drivers/test_template_definition.py

@@ -528,6 +528,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         autoscaler_tag = mock_cluster.labels.get('autoscaler_tag')
         min_node_count = mock_cluster.labels.get('min_node_count')
         max_node_count = mock_cluster.labels.get('max_node_count')
+        master_image = mock_cluster_template.image_id
+        minion_image = mock_cluster_template.image_id
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -600,6 +602,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'min_node_count': min_node_count,
             'max_node_count': max_node_count,
             'traefik_ingress_controller_tag': traefik_ingress_controller_tag,
+            'master_image': master_image,
+            'minion_image': minion_image,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -923,6 +927,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         autoscaler_tag = mock_cluster.labels.get('autoscaler_tag')
         min_node_count = mock_cluster.labels.get('min_node_count')
         max_node_count = mock_cluster.labels.get('max_node_count')
+        master_image = mock_cluster_template.image_id
+        minion_image = mock_cluster_template.image_id
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -997,6 +1003,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'min_node_count': min_node_count,
             'max_node_count': max_node_count,
             'traefik_ingress_controller_tag': traefik_ingress_controller_tag,
+            'master_image': master_image,
+            'minion_image': minion_image,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

releasenotes/notes/k8s-fedora-atomic-rolling-upgrade-3d8edcdd91fa1529.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Now the fedora atomic Kubernetes driver can support rolling upgrade for k8s
+    version change or the image change. User can call command
+    `openstack coe cluster upgrade <cluster ID> <new cluster template ID>` to
+    upgrade current cluster to the new version defined in the new cluster
+    template. At this moment, only the image change and the kube_tag change
+    are supported.
+issues:
+  - |
+    There is a known issue when doing image(operating system) upgrade for k8s
+    cluster. Because when doing image change for a server resource, Heat will
+    trigger the Nova rebuild to rebuild the instnace and there is no chance to
+    call kubectl drain to drain the node, so there could be a very minior
+    downtime when doing(starting to do) the rebuild and meanwhile a request
+    is routed to that node.