Merge "Add calico-node on k8s master node"

magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh

@@ -107,11 +107,17 @@ spec:
         k8s-app: calico-node
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: |
-          [{"key": "dedicated", "value": "master", "effect": "NoSchedule" },
-           {"key":"CriticalAddonsOnly", "operator":"Exists"}]
     spec:
       hostNetwork: true
+      tolerations:
+        # Make sure calico/node gets scheduled on all nodes.
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
       serviceAccountName: calico-node
       # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -5,6 +5,15 @@
 echo "configuring kubernetes (master)"
 
 _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
+
+# TODO(flwang): We should revisit this part to figure out if it's possible to
+# only run the calico-node container as a systemd service before starting the
+# minion nodes.
+if [ "$NETWORK_DRIVER" = "calico" ]; then
+    mkdir -p /opt/cni
+    _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
+    atomic install --storage ostree --system --set=ADDTL_MOUNTS=${_addtl_mounts} --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
+fi
 atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
 atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
@@ -74,7 +83,7 @@ sed -i '
 sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler
 
 HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
-KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${HOSTNAME_OVERRIDE}"
+KUBELET_ARGS="--register-node=true --register-schedulable=false --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${HOSTNAME_OVERRIDE}"
 KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
 KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
 
@@ -84,3 +93,60 @@ sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
 if [ -n "${INSECURE_REGISTRY_URL}" ]; then
     echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
 fi
+
+if [ "$NETWORK_DRIVER" = "calico" ]; then
+    KUBELET_ARGS="${KUBELET_ARGS} --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --register-with-taints=CriticalAddonsOnly=True:NoSchedule,dedicated=master:NoSchedule"
+
+    KUBELET_KUBECONFIG=/etc/kubernetes/kubelet-config.yaml
+    HOSTNAME_OVERRIDE=$(hostname --short | sed 's/\.novalocal//')
+    cat << EOF >> ${KUBELET_KUBECONFIG}
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority: ${CERT_DIR}/ca.crt
+    server: http://127.0.0.1:8080
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: system:node:${HOSTNAME_OVERRIDE}
+  name: default
+current-context: default
+kind: Config
+preferences: {}
+users:
+- name: system:node:${HOSTNAME_OVERRIDE}
+  user:
+    as-user-extra: {}
+    client-certificate: ${CERT_DIR}/server.crt
+    client-key: ${CERT_DIR}/server.key
+EOF
+
+    cat > /etc/kubernetes/get_require_kubeconfig.sh <<EOF
+#!/bin/bash
+
+KUBE_VERSION=\$(kubelet --version | awk '{print \$2}')
+min_version=v1.8.0
+if [[ "\${min_version}" != \$(echo -e "\${min_version}\n\${KUBE_VERSION}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "\${KUBE_VERSION}" != "devel" ]]; then
+    echo "--require-kubeconfig"
+fi
+EOF
+    chmod +x /etc/kubernetes/get_require_kubeconfig.sh
+
+    KUBELET_ARGS="${KUBELET_ARGS} --client-ca-file=${CERT_DIR}/ca.crt --tls-cert-file=${CERT_DIR}/kubelet.crt --tls-private-key-file=${CERT_DIR}/kubelet.key --kubeconfig ${KUBELET_KUBECONFIG}"
+
+    # specified cgroup driver
+    KUBELET_ARGS="${KUBELET_ARGS} --cgroup-driver=systemd"
+
+    if [ -z "${KUBE_NODE_IP}" ]; then
+        KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
+    fi
+
+    KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
+
+    sed -i '
+    /^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
+    /^KUBELET_HOSTNAME=/ s/=.*/=""/
+    /^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
+' /etc/kubernetes/kubelet
+fi

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -10,6 +10,19 @@ _addtl_mounts=''
 if [ "$NETWORK_DRIVER" = "calico" ]; then
     mkdir -p /opt/cni
     _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]}'
+
+    if [ "`systemctl status NetworkManager.service | grep -o "Active: active"`" = "Active: active" ]; then
+        CALICO_NM=/etc/NetworkManager/conf.d/calico.conf
+        [ -f ${CALICO_NM} ] || {
+        echo "Writing File: $CALICO_NM"
+        mkdir -p $(dirname ${CALICO_NM})
+        cat << EOF > ${CALICO_NM}
+[keyfile]
+unmanaged-devices=interface-name:cali*;interface-name:tunl*
+EOF
+}
+        systemctl restart NetworkManager
+    fi
 fi
 
 atomic install --storage ostree --system --system-package=no --set=ADDTL_MOUNTS=${_addtl_mounts} --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}

magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh

@@ -19,3 +19,9 @@ for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler
     systemctl enable $service
     systemctl --no-block start $service
 done
+
+if [ "$NETWORK_DRIVER" = "calico" ]; then
+    echo "activating service kubelet"
+    systemctl enable kubelet
+    systemctl start kubelet
+fi

magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh

@@ -65,14 +65,16 @@ sans="${sans},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,D
 
 cert_dir=/etc/kubernetes/certs
 mkdir -p "$cert_dir"
-
 CA_CERT=$cert_dir/ca.crt
-SERVER_CERT=$cert_dir/server.crt
-SERVER_CSR=$cert_dir/server.csr
-SERVER_KEY=$cert_dir/server.key
 
-#Get a token by user credentials and trust
-auth_json=$(cat << EOF
+function generate_certificates {
+    _CERT=$cert_dir/${1}.crt
+    _CSR=$cert_dir/${1}.csr
+    _KEY=$cert_dir/${1}.key
+    _CONF=$2
+
+    #Get a token by user credentials and trust
+    auth_json=$(cat << EOF
 {
     "auth": {
         "identity": {
@@ -91,16 +93,35 @@ auth_json=$(cat << EOF
 EOF
 )
 
-content_type='Content-Type: application/json'
-url="$AUTH_URL/auth/tokens"
-USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \
-    | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
+    content_type='Content-Type: application/json'
+    url="$AUTH_URL/auth/tokens"
+    USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \
+        | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
 
-# Get CA certificate for this cluster
-curl $VERIFY_CA -X GET \
-    -H "X-Auth-Token: $USER_TOKEN" \
-    -H "OpenStack-API-Version: container-infra latest" \
-    $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
+    # Get CA certificate for this cluster
+    curl $VERIFY_CA -X GET \
+        -H "X-Auth-Token: $USER_TOKEN" \
+        -H "OpenStack-API-Version: container-infra latest" \
+        $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
+
+    # Generate server's private key and csr
+    openssl genrsa -out "${_KEY}" 4096
+    chmod 400 "${_KEY}"
+    openssl req -new -days 1000 \
+            -key "${_KEY}" \
+            -out "${_CSR}" \
+            -reqexts req_ext \
+            -config "${_CONF}"
+
+    # Send csr to Magnum to have it signed
+    csr_req=$(python -c "import json; fp = open('${_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()")
+    curl $VERIFY_CA -X POST \
+        -H "X-Auth-Token: $USER_TOKEN" \
+        -H "OpenStack-API-Version: container-infra latest" \
+        -H "Content-Type: application/json" \
+        -d "$csr_req" \
+        $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${_CERT}
+}
 
 # Create config for server's csr
 cat > ${cert_dir}/server.conf <<EOF
@@ -115,23 +136,28 @@ subjectAltName = ${sans}
 extendedKeyUsage = clientAuth,serverAuth
 EOF
 
-# Generate server's private key and csr
-openssl genrsa -out "${SERVER_KEY}" 4096
-chmod 400 "${SERVER_KEY}"
-openssl req -new -days 1000 \
-        -key "${SERVER_KEY}" \
-        -out "${SERVER_CSR}" \
-        -reqexts req_ext \
-        -config "${cert_dir}/server.conf"
+#Kubelet Certs
+INSTANCE_NAME=$(hostname --short | sed 's/\.novalocal//')
+cat > ${cert_dir}/kubelet.conf <<EOF
+[req]
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+prompt = no
+[req_distinguished_name]
+CN = system:node:${INSTANCE_NAME}
+O=system:nodes
+OU=OpenStack/Magnum
+C=US
+ST=TX
+L=Austin
+[req_ext]
+subjectAltName = ${sans}
+keyUsage=critical,digitalSignature,keyEncipherment
+extendedKeyUsage=clientAuth,serverAuth
+EOF
 
-# Send csr to Magnum to have it signed
-csr_req=$(python -c "import json; fp = open('${SERVER_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()")
-curl $VERIFY_CA -X POST \
-    -H "X-Auth-Token: $USER_TOKEN" \
-    -H "OpenStack-API-Version: container-infra latest" \
-    -H "Content-Type: application/json" \
-    -d "$csr_req" \
-    $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}
+generate_certificates server ${cert_dir}/server.conf
+generate_certificates kubelet ${cert_dir}/kubelet.conf
 
 # Common certs and key are created for both etcd and kubernetes services.
 # Both etcd and kube user should have permission to access the certs and key.
@@ -140,6 +166,6 @@ usermod -a -G kube_etcd etcd
 usermod -a -G kube_etcd kube
 chmod 550 "${cert_dir}"
 chown -R kube:kube_etcd "${cert_dir}"
-chmod 440 $SERVER_KEY
+chmod 440 $cert_dir/server.key
 mkdir -p /etc/etcd/certs
 cp ${cert_dir}/* /etc/etcd/certs

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -527,6 +527,9 @@ resources:
         - protocol: tcp
           port_range_min: 6443
           port_range_max: 6443
+        - protocol: tcp
+          port_range_min: 10250
+          port_range_max: 10250
         - protocol: tcp
           port_range_min: 30000
           port_range_max: 32767