From c270539b16feac4d31f94faf4ab6d23e96e1d70e Mon Sep 17 00:00:00 2001
From: ArchiFleKs <kevin.lefevre@osones.io>
Date: Fri, 10 Feb 2017 12:27:24 +0100
Subject: [PATCH] Prepare Kubelet for multiple container runtime

This change introduces default recommended values for Kubelet on CoreOS:
- Usage of CNI (Container Networking Interface) with Flannel
- Update deprecated Kubelet Args (--config)
- Bind mount recommended CoreOS folders in Kubelet

It also introduces a new parameter: CONTAINER_RUNTIME which will allow to
switch between rkt and docker as container runtime. For now only docker
is used.

Partially-Implements: blueprint coreos-best-pratice
Change-Id: I1db1c3c06198b41098472f5c28405c533b91b41e
---
 .../fragments/enable-kubelet-master.yaml      | 38 ++++++++++++++++++-
 .../fragments/enable-kubelet-minion.yaml      | 38 ++++++++++++++++++-
 .../enable-network-service-client.yaml        | 12 ++++++
 .../fragments/enable-network-service.yaml     | 12 ++++++
 .../fragments/write-heat-params-master.yaml   |  1 +
 .../fragments/write-heat-params.yaml          |  1 +
 .../k8s_coreos_v1/templates/kubecluster.yaml  | 10 +++++
 .../k8s_coreos_v1/templates/kubemaster.yaml   |  6 +++
 .../k8s_coreos_v1/templates/kubeminion.yaml   |  6 +++
 9 files changed, 120 insertions(+), 4 deletions(-)

diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml
index b7f62e7287..2997a068fe 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml
@@ -31,27 +31,61 @@ write_files:
           INSECURE_REGISTRY_ARGS=""
       fi
 
+      uuid_file="/var/run/kubelet-pod.uuid"
       CONF_FILE=/etc/systemd/system/kubelet.service
       cat > $CONF_FILE <<EOF
       [Service]
       Environment=KUBELET_VERSION=${KUBE_VERSION}
       Environment=KUBELET_ACI=${HYPERKUBE_IMAGE_REPO}
+      Environment="RKT_OPTS=--uuid-file-save=${uuid_file} \
+        --volume dns,kind=host,source=/etc/resolv.conf \
+        --mount volume=dns,target=/etc/resolv.conf \
+        --volume rkt,kind=host,source=/opt/bin/host-rkt \
+        --mount volume=rkt,target=/usr/bin/rkt \
+        --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
+        --mount volume=var-lib-rkt,target=/var/lib/rkt \
+        --volume stage,kind=host,source=/tmp \
+        --mount volume=stage,target=/tmp \
+        --volume var-log,kind=host,source=/var/log \
+        --mount volume=var-log,target=/var/log"
+      ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
+      ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
+      ExecStartPre=/usr/bin/mkdir -p /var/log/containers
+      ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file}
       ExecStart=/usr/lib/coreos/kubelet-wrapper \
         --api-servers=http://127.0.0.1:8080 \
-        --address=0.0.0.0 \
+        --cni-conf-dir=/etc/kubernetes/cni/net.d \
+        --network-plugin=cni \
         --register-node=true \
+        --container-runtime=${CONTAINER_RUNTIME}
         --register-schedulable=false \
         --allow-privileged=true \
-        --config=/etc/kubernetes/manifests \
+        --pod-manifest-path=/etc/kubernetes/manifests \
         --hostname-override=${KUBE_NODE_IP} \
         --logtostderr=true \
         --v=0 \
         ${INSECURE_REGISTRY_ARGS}
+      ExecStop=-/usr/bin/rkt stop --uuid-file=${uuid_file}
       Restart=always
       RestartSec=10
       [Install]
       WantedBy=multi-user.target
       EOF
 
+      TEMPLATE=/opt/bin/host-rkt
+      mkdir -p $(dirname $TEMPLATE)
+      cat << EOF > $TEMPLATE
+      #!/bin/sh
+      # This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+      # through this rkt wrapper. It essentially enters the host mount namespace
+      # (which it is already in) only for the purpose of breaking out of the chroot
+      # before calling rkt. It makes things like rkt gc work and avoids bind mounting
+      # in certain rkt filesystem dependancies into the kubelet rootfs. This can
+      # eventually be obviated when the write-api stuff gets upstream and rkt gc is
+      # through the api-server. Related issue:
+      # https://github.com/coreos/rkt/issues/2878
+      exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
+      EOF
+
       systemctl enable kubelet
       systemctl --no-block start kubelet
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml
index cfd84b2fae..b09dc9e2e7 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml
@@ -43,17 +43,35 @@ write_files:
       fi
       KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
 
+      uuid_file="/var/run/kubelet-pod.uuid"
       CONF_FILE=/etc/systemd/system/kubelet.service
       cat > $CONF_FILE <<EOF
       [Service]
       Environment=KUBELET_VERSION=${KUBE_VERSION}
       Environment=KUBELET_ACI=${HYPERKUBE_IMAGE_REPO}
+      Environment="RKT_OPTS=--uuid-file-save=${uuid_file} \
+        --volume dns,kind=host,source=/etc/resolv.conf \
+        --mount volume=dns,target=/etc/resolv.conf \
+        --volume rkt,kind=host,source=/opt/bin/host-rkt \
+        --mount volume=rkt,target=/usr/bin/rkt \
+        --volume var-lib-rkt,kind=host,source=/var/lib/rkt \
+        --mount volume=var-lib-rkt,target=/var/lib/rkt \
+        --volume stage,kind=host,source=/tmp \
+        --mount volume=stage,target=/tmp \
+        --volume var-log,kind=host,source=/var/log \
+        --mount volume=var-log,target=/var/log"
+      ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
+      ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
+      ExecStartPre=/usr/bin/mkdir -p /var/log/containers
+      ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file}
       ExecStart=/usr/lib/coreos/kubelet-wrapper \
         --api-servers=${KUBE_MASTER_URI} \
-        --address=0.0.0.0 \
+        --cni-conf-dir=/etc/kubernetes/cni/net.d \
+        --network-plugin=cni \
+        --container-runtime=${CONTAINER_RUNTIME} \
         --register-node=true \
         --allow-privileged=true \
-        --config=/etc/kubernetes/manifests \
+        --pod-manifest-path=/etc/kubernetes/manifests \
         --hostname-override=${KUBE_NODE_IP} \
         --logtostderr=true \
         --v=0 \
@@ -64,9 +82,25 @@ write_files:
         ${INSECURE_REGISTRY_ARGS}
       Restart=always
       RestartSec=10
+      ExecStop=-/usr/bin/rkt stop --uuid-file=${uuid_file}
       [Install]
       WantedBy=multi-user.target
       EOF
 
+      TEMPLATE=/opt/bin/host-rkt
+      mkdir -p $(dirname $TEMPLATE)
+      cat << EOF > $TEMPLATE
+      #!/bin/sh
+      # This is bind mounted into the kubelet rootfs and all rkt shell-outs go
+      # through this rkt wrapper. It essentially enters the host mount namespace
+      # (which it is already in) only for the purpose of breaking out of the chroot
+      # before calling rkt. It makes things like rkt gc work and avoids bind mounting
+      # in certain rkt filesystem dependancies into the kubelet rootfs. This can
+      # eventually be obviated when the write-api stuff gets upstream and rkt gc is
+      # through the api-server. Related issue:
+      # https://github.com/coreos/rkt/issues/2878
+      exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@"
+      EOF
+
       systemctl enable kubelet
       systemctl --no-block start kubelet
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service-client.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service-client.yaml
index fbed975254..0c61f16c02 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service-client.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service-client.yaml
@@ -68,5 +68,17 @@ write_files:
       After=flanneld.service
       EOF
 
+      CNI=/etc/kubernetes/cni/net.d/10-flannel.conf
+      mkdir -p $(dirname $CNI)
+      cat << EOF > $CNI
+      {
+          "name": "podnet",
+          "type": "flannel",
+          "delegate": {
+              "isDefaultGateway": true
+          }
+      }
+      EOF
+
       systemctl enable flanneld
       systemctl --no-block start flanneld
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service.yaml
index 800f532803..8df6ffbbc6 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-network-service.yaml
@@ -68,5 +68,17 @@ write_files:
       After=flanneld.service
       EOF
 
+      CNI=/etc/kubernetes/cni/net.d/10-flannel.conf
+      mkdir -p $(dirname $CNI)
+      cat << EOF > $CNI
+      {
+          "name": "podnet",
+          "type": "flannel",
+          "delegate": {
+              "isDefaultGateway": true
+          }
+      }
+      EOF
+
       systemctl enable flanneld
       systemctl --no-block start flanneld
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml
index 9d70465b9d..e8716cb22d 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml
@@ -42,3 +42,4 @@ write_files:
       KUBE_CERTS_PATH="$KUBE_CERTS_PATH"
       HOST_CERTS_PATH="$HOST_CERTS_PATH"
       HYPERKUBE_IMAGE_REPO="$HYPERKUBE_IMAGE_REPO"
+      CONTAINER_RUNTIME="$CONTAINER_RUNTIME"
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml
index fe59185d8a..4c3f1a4e24 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml
@@ -43,3 +43,4 @@ write_files:
       KUBE_CERTS_PATH="$KUBE_CERTS_PATH"
       HOST_CERTS_PATH="$HOST_CERTS_PATH"
       HYPERKUBE_IMAGE_REPO="$HYPERKUBE_IMAGE_REPO"
+      CONTAINER_RUNTIME="$CONTAINER_RUNTIME"
diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
index 034531939e..35c46653e9 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
@@ -242,6 +242,14 @@ parameters:
       - allowed_pattern: "^$|.*/"
     default: ""
 
+  container_runtime:
+    type: string
+    description: >
+      Container runtime to use with Kubernetes.
+    default: "docker"
+    constraints:
+      - allowed_values: ["docker"]
+
 resources:
 
   ######################################################################
@@ -398,6 +406,7 @@ resources:
           auth_url: {get_param: auth_url}
           hyperkube_image: {get_param: hyperkube_image}
           insecure_registry_url: {get_param: insecure_registry_url}
+          container_runtime: {get_param: container_runtime}
 
   ######################################################################
   #
@@ -443,6 +452,7 @@ resources:
           auth_url: {get_param: auth_url}
           hyperkube_image: {get_param: hyperkube_image}
           insecure_registry_url: {get_param: insecure_registry_url}
+          container_runtime: {get_param: container_runtime}
 
 outputs:
 
diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
index 503d9717d5..7e27f11f32 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
@@ -173,6 +173,11 @@ parameters:
     type: string
     description: insecure registry url
 
+  container_runtime:
+    type: string
+    description: >
+      Container runtime to use with Kubernetes.
+
 resources:
 
   master_wait_handle:
@@ -250,6 +255,7 @@ resources:
                   insecure_registry_url: { get_param: insecure_registry_url }
                   hyperkube_image: { get_param: hyperkube_image }
             "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
+            "$CONTAINER_RUNTIME": {get_param: container_runtime}
 
   configure_etcd:
     type: OS::Heat::SoftwareConfig
diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
index f910da7f4b..a394127e2b 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
@@ -128,6 +128,11 @@ parameters:
     type: string
     description: insecure registry url
 
+  container_runtime:
+    type: string
+    description: >
+      Container runtime to use with Kubernetes.
+
 resources:
 
   minion_wait_handle:
@@ -182,6 +187,7 @@ resources:
                   insecure_registry_url: { get_param: insecure_registry_url }
                   hyperkube_image: { get_param: hyperkube_image }
             "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
+            "$CONTAINER_RUNTIME": {get_param: container_runtime}
 
   write_kubeconfig:
     type: OS::Heat::SoftwareConfig