[suse] Add TLS support for k8s_opensuse_v1 driver

Partial-bug: #1622949
Change-Id: I4e8e8bddd8456c83f1bf1719256ddfca5a2723c4

contrib/drivers/k8s_opensuse_v1/templates/fragments/configure-kubernetes-master.sh

@@ -14,15 +14,30 @@ fi
 # Setting correct permissions for Kubernetes files
 chown -R kube:kube /var/lib/kubernetes
 
+KUBE_API_ARGS="--service-account-key-file=$SERVICE_ACCOUNT_KEY --runtime_config=api/all=true"
+
+if [ "$TLS_DISABLED" == "True" ]; then
+    sed -i '
+        /^# KUBE_API_PORT=/ s|.*|KUBE_API_PORT="--port=8080 --insecure-port='"$KUBE_API_PORT"'"|
+    ' /etc/kubernetes/apiserver
+else
+    # insecure port is used internaly
+    sed -i '
+        /^# KUBE_API_PORT=/ s|.*|KUBE_API_PORT="--port=8080 --insecure-port=8080 --secure-port='"$KUBE_API_PORT"'"|
+    ' /etc/kubernetes/apiserver
+    KUBE_API_ARGS="$KUBE_API_ARGS --tls_cert_file=/etc/kubernetes/ssl/server.crt"
+    KUBE_API_ARGS="$KUBE_API_ARGS --tls_private_key_file=/etc/kubernetes/ssl/server.key"
+    KUBE_API_ARGS="$KUBE_API_ARGS --client_ca_file=/etc/kubernetes/ssl/ca.crt"
+fi
+
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s|=.*|="--allow-privileged='"$KUBE_ALLOW_PRIV"'"|
 ' /etc/kubernetes/config
 
 sed -i '
-    /^KUBE_API_ADDRESS=/ s|=.*|="--advertise-address='"$KUBE_NODE_IP"' --insecure-bind-address=0.0.0.0"|
+    /^KUBE_API_ADDRESS=/ s|=.*|="--advertise-address='"$KUBE_NODE_IP"' --insecure-bind-address=0.0.0.0 --bind_address=0.0.0.0"|
-    /^KUBE_API_PORT=/ s|=.*|="--insecure-port='"$KUBE_API_PORT"'"|
     /^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"$PORTAL_NETWORK_CIDR"'"|
-    /^KUBE_API_ARGS=/ s|=.*|="--service-account-key-file='"$SERVICE_ACCOUNT_KEY"' --runtime-config=api\/all=true"|
+    /^KUBE_API_ARGS=/ s|=.*|="'"$KUBE_API_ARGS"'"|
     /^KUBE_ETCD_SERVERS=/ s/=.*/="--etcd-servers=http:\/\/127.0.0.1:2379"/
     /^KUBE_ADMISSION_CONTROL=/ s/=.*/="--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota"/
 ' /etc/kubernetes/apiserver

contrib/drivers/k8s_opensuse_v1/templates/fragments/configure-kubernetes-minion.sh

@@ -13,7 +13,7 @@ if [ "$TLS_DISABLED" == "True" ]; then
     KUBE_CONFIG=""
 else
     KUBE_PROTOCOL="https"
-    KUBE_CONFIG="--kubeconfig=/srv/kubernetes/kubeconfig.yaml"
+    KUBE_CONFIG="--kubeconfig=/etc/kubernetes/kubeconfig.yaml"
 fi
 
 KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
@@ -32,7 +32,7 @@ sed -i '
 ' /etc/kubernetes/kubelet
 
 sed -i '
-    /^KUBE_PROXY_ARGS=/ s/=.*/="--proxy-mode=iptables"/
+    /^KUBE_PROXY_ARGS=/ s|=.*|="--proxy-mode=iptables '"$KUBE_CONFIG"'"|
 ' /etc/kubernetes/proxy
 
 cat >> /etc/environment <<EOF

contrib/drivers/k8s_opensuse_v1/templates/fragments/make-cert-client.sh

@@ -0,0 +1,118 @@
+#!/bin/sh
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/sysconfig/heat-params
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [ "$TLS_DISABLED" == "True" ]; then
+    exit 0
+fi
+
+cert_dir=/etc/kubernetes/ssl
+
+mkdir -p "$cert_dir"
+
+CA_CERT=$cert_dir/ca.crt
+CLIENT_CERT=$cert_dir/client.crt
+CLIENT_CSR=$cert_dir/client.csr
+CLIENT_KEY=$cert_dir/client.key
+KUBE_MASTER_URI="https://$KUBE_MASTER_IP:$KUBE_API_PORT"
+
+#Get a token by user credentials and trust
+auth_json=$(cat << EOF
+{
+    "auth": {
+        "identity": {
+            "methods": [
+                "password"
+            ],
+            "password": {
+                "user": {
+                    "id": "$TRUSTEE_USER_ID",
+                    "password": "$TRUSTEE_PASSWORD"
+                }
+            }
+        },
+        "scope": {
+            "OS-TRUST:trust": {
+                "id": "$TRUST_ID"
+            }
+        }
+    }
+}
+EOF
+)
+
+#trust is introduced in Keystone v3 version
+AUTH_URL=${AUTH_URL/v2.0/v3}
+content_type='Content-Type: application/json'
+url="$AUTH_URL/auth/tokens"
+USER_TOKEN=`curl -s -i -X POST -H "$content_type" -d "$auth_json" $url \
+    | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
+
+# Get CA certificate for this cluster
+curl -X GET \
+  -H "X-Auth-Token: $USER_TOKEN" \
+  $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > $CA_CERT
+
+# Create config for client's csr
+cat > ${cert_dir}/client.conf <<EOF
+[req]
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+prompt = no
+[req_distinguished_name]
+CN = kubernetes.invalid
+[req_ext]
+keyUsage=critical,digitalSignature,keyEncipherment
+extendedKeyUsage=clientAuth
+subjectAltName=dirName:kubelet,dirName:kubeproxy
+[kubelet]
+CN=kubelet
+[kubeproxy]
+CN=kube-proxy
+EOF
+
+# Generate client's private key and csr
+openssl genrsa -out "${CLIENT_KEY}" 4096
+chmod 600 ${CLIENT_KEY}
+openssl req -new -days 1000 \
+        -key "${CLIENT_KEY}" \
+        -out "${CLIENT_CSR}" \
+        -reqexts req_ext \
+        -config "${cert_dir}/client.conf"
+
+# Send csr to Magnum to have it signed
+csr_req=$(python -c "import json; fp = open('${CLIENT_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()")
+curl -X POST \
+    -H "X-Auth-Token: $USER_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$csr_req" \
+    $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CLIENT_CERT}
+
+chmod 700 ${cert_dir}
+chmod 600 ${cert_dir}/*
+chown -R kube:kube ${cert_dir}
+
+sed -i '
+  s|CA_CERT|'"$CA_CERT"'|
+  s|CLIENT_CERT|'"$CLIENT_CERT"'|
+  s|CLIENT_KEY|'"$CLIENT_KEY"'|
+  s|KUBE_MASTER_URI|'"$KUBE_MASTER_URI"'|
+' /etc/kubernetes/kubeconfig.yaml

contrib/drivers/k8s_opensuse_v1/templates/fragments/make-cert.sh

@@ -0,0 +1,127 @@
+#!/bin/sh
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/sysconfig/heat-params
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [ "$TLS_DISABLED" == "True" ]; then
+  exit 0
+fi
+
+if [[ -z "${KUBE_NODE_PUBLIC_IP}" ]]; then
+    KUBE_NODE_PUBLIC_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+fi
+if [[ -z "${KUBE_NODE_IP}" ]]; then
+    KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
+fi
+
+sans="IP:${KUBE_NODE_PUBLIC_IP},IP:${KUBE_NODE_IP}"
+if [ "${KUBE_NODE_PUBLIC_IP}" != "${KUBE_API_PUBLIC_ADDRESS}" ] \
+  && [ -n "${KUBE_API_PUBLIC_ADDRESS}" ]; then
+    sans="${sans},IP:${KUBE_API_PUBLIC_ADDRESS}"
+fi
+if [ "${KUBE_NODE_IP}" != "${KUBE_API_PRIVATE_ADDRESS}" ] \
+  && [ -n "${KUBE_API_PRIVATE_ADDRESS}" ]; then
+    sans="${sans},IP:${KUBE_API_PRIVATE_ADDRESS}"
+fi
+MASTER_HOSTNAME=${MASTER_HOSTNAME:-}
+if [[ -n "${MASTER_HOSTNAME}" ]]; then
+    sans="${sans},DNS:${MASTER_HOSTNAME}"
+fi
+sans="${sans},IP:127.0.0.1"
+
+cert_dir=/etc/kubernetes/ssl
+
+mkdir -p "$cert_dir"
+
+CA_CERT=$cert_dir/ca.crt
+SERVER_CERT=$cert_dir/server.crt
+SERVER_CSR=$cert_dir/server.csr
+SERVER_KEY=$cert_dir/server.key
+
+#Get a token by user credentials and trust
+auth_json=$(cat << EOF
+{
+    "auth": {
+        "identity": {
+            "methods": [
+                "password"
+            ],
+            "password": {
+                "user": {
+                    "id": "$TRUSTEE_USER_ID",
+                    "password": "$TRUSTEE_PASSWORD"
+                }
+            }
+        },
+        "scope": {
+            "OS-TRUST:trust": {
+                "id": "$TRUST_ID"
+            }
+        }
+    }
+}
+EOF
+)
+
+#trust is introduced in Keystone v3 version
+AUTH_URL=${AUTH_URL/v2.0/v3}
+content_type='Content-Type: application/json'
+url="$AUTH_URL/auth/tokens"
+USER_TOKEN=`curl -s -i -X POST -H "$content_type" -d "$auth_json" $url \
+    | grep X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
+
+# Get CA certificate for this cluster
+curl -X GET \
+  -H "X-Auth-Token: $USER_TOKEN" \
+  $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
+
+# Create config for server's csr
+cat > ${cert_dir}/server.conf <<EOF
+[req]
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+prompt = no
+[req_distinguished_name]
+CN = kubernetes.invalid
+[req_ext]
+subjectAltName = ${sans}
+extendedKeyUsage = clientAuth,serverAuth
+EOF
+
+# Generate server's private key and csr
+openssl genrsa -out "${SERVER_KEY}" 4096
+chmod 600 ${SERVER_KEY}
+openssl req -new -days 1000 \
+        -key "${SERVER_KEY}" \
+        -out "${SERVER_CSR}" \
+        -reqexts req_ext \
+        -config "${cert_dir}/server.conf"
+
+# Send csr to Magnum to have it signed
+csr_req=$(python -c "import json; fp = open('${SERVER_CSR}'); print json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()}); fp.close()")
+curl -X POST \
+    -H "X-Auth-Token: $USER_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$csr_req" \
+    $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}
+
+chmod 700 ${cert_dir}
+chmod 600 ${cert_dir}/*
+chown -R kube:kube ${cert_dir}

contrib/drivers/k8s_opensuse_v1/templates/fragments/write-heat-params-master.yaml

@@ -8,6 +8,11 @@ write_files:
       KUBE_NODE_IP="$KUBE_NODE_IP"
       KUBE_API_PORT="$KUBE_API_PORT"
       KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
+      KUBE_MASTER_IPS="$KUBE_MASTER_IPS"
+      KUBE_MINION_IPS="$KUBE_MINION_IPS"
+      KUBE_NODE_PUBLIC_IP="$KUBE_NODE_PUBLIC_IP"
+      KUBE_NODE_IP="$KUBE_NODE_IP"
+      KUBE_NODE_NAME="$KUBE_NODE_NAME"
       NETWORK_DRIVER="$NETWORK_DRIVER"
       FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
       FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"
@@ -27,3 +32,6 @@ write_files:
       MAGNUM_URL="$MAGNUM_URL"
       SYSTEM_PODS_INITIAL_DELAY="$SYSTEM_PODS_INITIAL_DELAY"
       SYSTEM_PODS_TIMEOUT="$SYSTEM_PODS_TIMEOUT"
+      TRUSTEE_USER_ID="$TRUSTEE_USER_ID"
+      TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD"
+      TRUST_ID="$TRUST_ID"

contrib/drivers/k8s_opensuse_v1/templates/fragments/write-heat-params-minion.yaml

@@ -30,3 +30,9 @@ write_files:
       HTTP_PROXY="$HTTP_PROXY"
       HTTPS_PROXY="$HTTPS_PROXY"
       NO_PROXY="$NO_PROXY"
+      AUTH_URL="$AUTH_URL"
+      TRUSTEE_USER_ID="$TRUSTEE_USER_ID"
+      TRUSTEE_USERNAME="$TRUSTEE_USERNAME"
+      TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD"
+      TRUSTEE_DOMAIN_ID="$TRUSTEE_DOMAIN_ID"
+      TRUST_ID="$TRUST_ID"

contrib/drivers/k8s_opensuse_v1/templates/fragments/write-kubeconfig.yaml

@@ -0,0 +1,25 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /etc/kubernetes/kubeconfig.yaml
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      apiVersion: v1
+      kind: Config
+      users:
+      - name: kubeclient
+        user:
+          client-certificate: CLIENT_CERT
+          client-key: CLIENT_KEY
+      clusters:
+      - name: kubernetes
+        cluster:
+          certificate-authority: CA_CERT
+          server: KUBE_MASTER_URI
+      contexts:
+      - context:
+          cluster: kubernetes
+          user: kubeclient
+        name: service-account-context
+      current-context: service-account-context

contrib/drivers/k8s_opensuse_v1/templates/kubecluster.yaml

@@ -548,6 +548,13 @@ resources:
           secgroup_base_id: {get_resource: secgroup_base}
           secgroup_kube_master_id: {get_resource: secgroup_kube_master}
           kube_master_id: 'kube-master%index%'
+          kube_master_ports: { get_attr: [kube_master_ports, refs] }
+          kube_master_ips: {get_attr: [kube_master_ports, fixed_ip]}
+          kube_master_ips_list: { list_join: ["|", {get_attr: [kube_master_ports, fixed_ip]} ] }
+          kube_minion_ips_list: { list_join: ["|", {get_attr: [kube_minion_ports, fixed_ip]} ] }
+          trustee_user_id: {get_param: trustee_user_id}
+          trustee_password: {get_param: trustee_password}
+          trust_id: {get_param: trust_id}
 
   ######################################################################
   #
@@ -600,6 +607,16 @@ resources:
           tls_disabled: {get_param: tls_disabled}
           secgroup_kube_minion_id: {get_resource: secgroup_kube_minion}
           kube_minion_id: 'kube-minion%index%'
+          kube_minion_ports: { get_attr: [kube_minion_ports, refs] }
+          kube_minion_ips: {get_attr: [kube_minion_ports, fixed_ip]}
+          kube_master_ips_list: { list_join: ["|", {get_attr: [kube_master_ports, fixed_ip]} ] }
+          kube_minion_ips_list: { list_join: ["|", {get_attr: [kube_minion_ports, fixed_ip]} ] }
+          auth_url: {get_param: auth_url}
+          trustee_user_id: {get_param: trustee_user_id}
+          trustee_username: {get_param: trustee_username}
+          trustee_password: {get_param: trustee_password}
+          trustee_domain_id: {get_param: trustee_domain_id}
+          trust_id: {get_param: trust_id}
 
 outputs:
 

contrib/drivers/k8s_opensuse_v1/templates/kubemaster.yaml

@@ -184,6 +184,20 @@ parameters:
     type: string
     description: ID of for kubernetes master.
 
+  trustee_user_id:
+    type: string
+    description: user id of the trustee
+
+  trustee_password:
+    type: string
+    description: password of the trustee
+    hidden: true
+
+  trust_id:
+    type: string
+    description: id of the trust which is used by the trustee
+    hidden: true
+
 resources:
 
   master_wait_handle:
@@ -214,6 +228,11 @@ resources:
             "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
             "$KUBE_API_PORT": {get_param: kubernetes_port}
             "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
+            "$KUBE_MASTER_IPS": {get_param: kube_master_ips_list}
+            "$KUBE_MINION_IPS": {get_param: kube_minion_ips_list}
+            "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
+            "$KUBE_NODE_IP": { "Fn::Select": [ { get_param: kube_master_index }, { get_param: kube_master_ips} ] }
+            "$KUBE_NODE_NAME": {get_param: kube_master_id}
             "$NETWORK_DRIVER": {get_param: network_driver}
             "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
             "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
@@ -236,6 +255,15 @@ resources:
             "$HTTP_PROXY": {get_param: http_proxy}
             "$HTTPS_PROXY": {get_param: https_proxy}
             "$NO_PROXY": {get_param: no_proxy}
+            "$TRUSTEE_USER_ID": {get_param: trustee_user_id}
+            "$TRUSTEE_PASSWORD": {get_param: trustee_password}
+            "$TRUST_ID": {get_param: trust_id}
+
+  make_cert:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/make-cert.sh}
 
   configure_etcd:
     type: OS::Heat::SoftwareConfig
@@ -284,6 +312,7 @@ resources:
     properties:
       parts:
         - config: {get_resource: write_heat_params}
+        - config: {get_resource: make_cert}
         - config: {get_resource: configure_etcd}
         - config: {get_resource: configure_flanneld}
         - config: {get_resource: create_kubernetes_user}

contrib/drivers/k8s_opensuse_v1/templates/kubeminion.yaml

@@ -172,6 +172,33 @@ parameters:
     type: string
     description: ID of for kubernetes minion.
 
+  auth_url:
+    type: string
+    description: >
+      url for kubernetes to authenticate before sending request to neutron
+
+  trustee_domain_id:
+    type: string
+    description: domain id of the trustee
+
+  trustee_user_id:
+    type: string
+    description: user id of the trustee
+
+  trustee_username:
+    type: string
+    description: username of the trustee
+
+  trustee_password:
+    type: string
+    description: password of the trustee
+    hidden: true
+
+  trust_id:
+    type: string
+    description: id of the trust which is used by the trustee
+    hidden: true
+
 resources:
 
   minion_wait_handle:
@@ -223,6 +250,24 @@ resources:
             "$HTTP_PROXY": {get_param: http_proxy}
             "$HTTPS_PROXY": {get_param: https_proxy}
             "$NO_PROXY": {get_param: no_proxy}
+            "$AUTH_URL": {get_param: auth_url}
+            "$TRUSTEE_DOMAIN_ID": {get_param: trustee_domain_id}
+            "$TRUSTEE_USER_ID": {get_param: trustee_user_id}
+            "$TRUSTEE_USERNAME": {get_param: trustee_username}
+            "$TRUSTEE_PASSWORD": {get_param: trustee_password}
+            "$TRUST_ID": {get_param: trust_id}
+
+  write_kubeconfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/write-kubeconfig.yaml}
+
+  make_cert:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/make-cert-client.sh}
 
   configure_flanneld:
     type: OS::Heat::SoftwareConfig
@@ -271,6 +316,8 @@ resources:
     properties:
       parts:
         - config: {get_resource: write_heat_params}
+        - config: {get_resource: write_kubeconfig}
+        - config: {get_resource: make_cert}
         - config: {get_resource: configure_flanneld}
         - config: {get_resource: configure_docker}
         - config: {get_resource: create_kubernetes_user}