Add TLS support in heat kubernetes

This patch modifies template to generate certificates and configure TLS
settings for kube-apiserver/kubelet/kube-proxy.

Co-Authored-By: Andrew Melton <andrew.melton@rackspace.com>
Partially-Implements: bp secure-kubernetes
Change-Id: I76b0f91f0c44f9880980e35c6b8856ea48ed3ce1

etc/magnum/magnum.conf.sample

@@ -720,22 +720,6 @@
 #admin_tenant_name = admin
 
 
-[kubernetes]
-
-#
-# From magnum
-#
-
-# Default protocol of k8s master endpoint (http or https). (string
-# value)
-#k8s_protocol = http
-
-# Default port of the k8s master endpoint. (unknown type)
-# Minimum value: 1
-# Maximum value: 65535
-#k8s_port = 8080
-
-
 [magnum_client]
 
 #

magnum/conductor/handlers/bay_conductor.py

@@ -215,9 +215,9 @@ class HeatPoller(object):
         self.context = self.openstack_client.context
         self.bay = bay
         self.attempts = 0
-        baymodel = conductor_utils.retrieve_baymodel(self.context, bay)
+        self.baymodel = conductor_utils.retrieve_baymodel(self.context, bay)
         self.template_def = TDef.get_template_definition(
-            'vm', baymodel.cluster_distro, baymodel.coe)
+            'vm', self.baymodel.cluster_distro, self.baymodel.coe)
 
     def poll_and_check(self):
         # TODO(yuanying): temporary implementation to update api_address,
@@ -238,7 +238,7 @@ class HeatPoller(object):
             raise loopingcall.LoopingCallDone()
         if (stack.stack_status in [bay_status.CREATE_COMPLETE,
                                    bay_status.UPDATE_COMPLETE]):
-            self.template_def.update_outputs(stack, self.bay)
+            self.template_def.update_outputs(stack, self.baymodel, self.bay)
 
             self.bay.status = stack.stack_status
             self.bay.status_reason = stack.stack_status_reason

magnum/conductor/k8s_api.py

@@ -12,27 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from oslo_config import cfg
-
-from magnum.common import config
 from magnum.common.pythonk8sclient.swagger_client import api_client
 from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
 from magnum.conductor import utils
-from magnum.i18n import _
-
-
-kubernetes_opts = [
-    cfg.StrOpt('k8s_protocol',
-               default='http',
-               help=_('Default protocol of k8s master endpoint '
-                      '(http or https).')),
-    cfg.Opt('k8s_port',
-            type=config.PORT_TYPE,
-            default=8080,
-            help=_('Default port of the k8s master endpoint.')),
-]
-
-cfg.CONF.register_opts(kubernetes_opts, group='kubernetes')
 
 
 class K8sAPI(apiv_api.ApivApi):
@@ -51,11 +33,7 @@ class K8sAPI(apiv_api.ApivApi):
         if hasattr(obj, 'bay_uuid'):
             obj = utils.retrieve_bay(context, obj)
 
-        params = {
+        return obj.api_address
-            'k8s_protocol': cfg.CONF.kubernetes.k8s_protocol,
-            'api_address': obj.api_address
-        }
-        return "%(k8s_protocol)s://%(api_address)s" % params
 
 
 def create_k8s_api(context, obj):

magnum/conductor/template_definition.py

@@ -29,6 +29,8 @@ from magnum.i18n import _LW
 
 LOG = logging.getLogger(__name__)
 
+KUBE_SECURE_PORT = '6443'
+KUBE_INSECURE_PORT = '8080'
 
 template_def_opts = [
     cfg.StrOpt('k8s_atomic_template_path',
@@ -133,7 +135,7 @@ class OutputMapping(object):
         self.bay_attr = bay_attr
         self.heat_output = heat_output
 
-    def set_output(self, stack, bay):
+    def set_output(self, stack, baymodel, bay):
         if self.bay_attr is None:
             return
 
@@ -278,7 +280,8 @@ class TemplateDefinition(object):
         self.param_mappings.append(param)
 
     def add_output(self, *args, **kwargs):
-        output = OutputMapping(*args, **kwargs)
+        mapping_type = kwargs.pop('mapping_type', OutputMapping)
+        output = mapping_type(*args, **kwargs)
         self.output_mappings.append(output)
 
     def get_output(self, *args, **kwargs):
@@ -325,9 +328,9 @@ class TemplateDefinition(object):
 
         return None
 
-    def update_outputs(self, stack, bay):
+    def update_outputs(self, stack, baymodel, bay):
         for output in self.output_mappings:
-            output.set_output(stack, bay)
+            output.set_output(stack, baymodel, bay)
 
     @abc.abstractproperty
     def template_path(self):
@@ -362,6 +365,28 @@ class BaseTemplateDefinition(TemplateDefinition):
         pass
 
 
+class K8sApiAddressOutputMapping(OutputMapping):
+
+    def set_output(self, stack, baymodel, bay):
+        # TODO(yuanying): port number is hardcoded, this will be fix
+        protocol = 'https'
+        port = KUBE_SECURE_PORT
+        if baymodel.tls_disabled:
+            protocol = 'http'
+            port = KUBE_INSECURE_PORT
+
+        output_value = self.get_output_value(stack)
+        params = {
+            'protocol': protocol,
+            'address': output_value,
+            'port': port,
+        }
+        output_value = "%(protocol)s://%(address)s:%(port)s" % params
+
+        if output_value is not None:
+            setattr(bay, self.bay_attr, output_value)
+
+
 class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
     """Kubernetes template for a Fedora Atomic VM."""
 
@@ -373,6 +398,9 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
 
     def __init__(self):
         super(AtomicK8sTemplateDefinition, self).__init__()
+        self.add_parameter('bay_uuid',
+                           bay_attr='uuid',
+                           param_type=str)
         self.add_parameter('master_flavor',
                            baymodel_attr='master_flavor_id')
         self.add_parameter('minion_flavor',
@@ -390,13 +418,13 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
                            required=True)
         self.add_parameter('network_driver',
                            baymodel_attr='network_driver')
-        # TODO(yuanying): Add below lines if apiserver_port parameter
+        self.add_parameter('tls_disabled',
-        # is supported
+                           baymodel_attr='tls_disabled',
-        # self.add_parameter('apiserver_port',
+                           required=True)
-        #                    baymodel_attr='apiserver_port')
 
         self.add_output('api_address',
-                        bay_attr='api_address')
+                        bay_attr='api_address',
+                        mapping_type=K8sApiAddressOutputMapping)
         self.add_output('kube_minions',
                         bay_attr=None)
         self.add_output('kube_minions_external',
@@ -433,6 +461,13 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
         extra_params['auth_url'] = context.auth_url.replace("v3", "v2")
         extra_params['username'] = context.user_name
         extra_params['tenant_name'] = context.tenant
+        extra_params['user_token'] = context.auth_token
+        osc = clients.OpenStackClients(context)
+        extra_params['magnum_url'] = osc.magnum_url()
+
+        if baymodel.tls_disabled:
+            extra_params['loadbalancing_protocol'] = 'HTTP'
+            extra_params['kubernetes_port'] = 8080
 
         for label in label_list:
             extra_params[label] = baymodel.labels.get(label)

magnum/opts.py

@@ -55,7 +55,5 @@ def list_opts():
         ('certificates',
             itertools.chain(magnum.common.cert_manager.cert_manager_opts,
                             local_cert_manager.local_cert_manager_opts,
-                            )),
+                            ))
-        ('kubernetes',
-            magnum.conductor.k8s_api.kubernetes_opts),
     ]

magnum/templates/heat-kubernetes/fragments/configure-kubernetes-master.sh

@@ -11,13 +11,31 @@ sed -i '
   /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
 ' /etc/kubernetes/config
 
+KUBE_API_ARGS="--runtime_config=api/all=true"
+if [ "$TLS_DISABLED" == "True" ]; then
+  KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0 --insecure-port=$KUBE_API_PORT"
+else
+  KUBE_API_ADDRESS="--bind_address=0.0.0.0 --secure-port=$KUBE_API_PORT"
+  # insecure port is used internaly
+  KUBE_API_ADDRESS="$KUBE_API_ADDRESS --insecure-port=8080"
+  KUBE_API_ARGS="$KUBE_API_ARGS --tls_cert_file=/srv/kubernetes/server.crt"
+  KUBE_API_ARGS="$KUBE_API_ARGS --tls_private_key_file=/srv/kubernetes/server.key"
+  KUBE_API_ARGS="$KUBE_API_ARGS --client_ca_file=/srv/kubernetes/ca.crt"
+fi
+
 sed -i '
-  /^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
+  /^KUBE_API_ADDRESS=/ s/=.*/='"${KUBE_API_ADDRESS}"'/
   /^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"$PORTAL_NETWORK_CIDR"'"|
-  /^KUBE_API_ARGS=/ s/KUBE_API_ARGS./#Uncomment the following line to disable Load Balancer feature\nKUBE_API_ARGS="--runtime_config=api\/all=true"\n#Uncomment the following line to enable Load Balancer feature\n#KUBE_API_ARGS="--runtime_config=api\/all=true --cloud_config=\/etc\/sysconfig\/kube_openstack_config --cloud_provider=openstack"/
+  /^KUBE_API_ARGS=/ s/KUBE_API_ARGS.//
   /^KUBE_ETCD_SERVERS=/ s/=.*/="--etcd_servers=http:\/\/127.0.0.1:2379"/
   /^KUBE_ADMISSION_CONTROL=/ s/=.*/=""/
 ' /etc/kubernetes/apiserver
+cat << _EOC_ >> /etc/kubernetes/apiserver
+#Uncomment the following line to disable Load Balancer feature
+KUBE_API_ARGS="$KUBE_API_ARGS"
+#Uncomment the following line to enable Load Balancer feature
+#KUBE_API_ARGS="$KUBE_API_ARGS --cloud_config=/etc/sysconfig/kube_openstack_config --cloud_provider=openstack"
+_EOC_
 
 sed -i '
   /^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/

magnum/templates/heat-kubernetes/fragments/configure-kubernetes-minion.sh

@@ -5,22 +5,31 @@
 echo "configuring kubernetes (minion)"
 
 ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
+KUBE_PROTOCOL="https"
+KUBE_CONFIG=""
+if [ "$TLS_DISABLED" == "True" ]; then
+    KUBE_PROTOCOL="http"
+else
+    KUBE_CONFIG="--kubeconfig=/srv/kubernetes/kubeconfig.yaml"
+fi
+KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
 
 sed -i '
 /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
 /^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://'"$ETCD_SERVER_IP"':2379"|
-/^KUBE_MASTER=/ s|=.*|="--master=http://'"$KUBE_MASTER_IP"':8080"|
+/^KUBE_MASTER=/ s|=.*|="--master='"$KUBE_MASTER_URI"'"|
 ' /etc/kubernetes/config
 
 sed -i '
 /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
 /^KUBELET_HOSTNAME=/ s/=.*/=""/
-/^KUBELET_API_SERVER=/ s|=.*|="--api_servers=http://'"$KUBE_MASTER_IP"':8080"|
+/^KUBELET_API_SERVER=/ s|=.*|="--api_servers='"$KUBE_MASTER_URI"'"|
+/^KUBELET_ARGS=/ s|=.*|='"$KUBE_CONFIG"'|
 ' /etc/kubernetes/kubelet
 
 sed -i '
-/^KUBE_MASTER=/ s/=.*/="--master='"$KUBE_MASTER_IP"':8080"/
+/^KUBE_PROXY_ARGS=/ s|=.*|='"$KUBE_CONFIG"'|
-' /etc/kubernetes/apiserver
+' /etc/kubernetes/proxy
 
 if [ "$NETWORK_DRIVER" == "flannel" ]; then
 sed -i '
@@ -29,10 +38,9 @@ sed -i '
 fi
 
 cat >> /etc/environment <<EOF
-KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
+KUBERNETES_MASTER=$KUBE_MASTER_URI
 EOF
 
 sed -i '/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=--storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=\/dev\/mapper\/docker-docker--pool --storage-opt dm.use_deferred_removal=true/' /etc/sysconfig/docker-storage
 
 hostname `hostname | sed 's/.novalocal//'`
-

magnum/templates/heat-kubernetes/fragments/make-cert-client.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+. /etc/sysconfig/heat-params
+
+if [ "$TLS_DISABLED" == "True" ]; then
+  exit 0
+fi
+
+cert_dir=/srv/kubernetes
+cert_conf_dir=${cert_dir}/conf
+cert_group=root
+
+mkdir -p "$cert_dir"
+mkdir -p "$cert_conf_dir"
+
+CA_CERT=$cert_dir/ca.crt
+CLIENT_CERT=$cert_dir/client.crt
+CLIENT_CSR=$cert_dir/client.csr
+CLIENT_KEY=$cert_dir/client.key
+
+# Get CA certificate for this bay
+curl -X GET \
+    -H "X-Auth-Token: $USER_TOKEN" \
+    $MAGNUM_URL/certificates/$BAY_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > $CA_CERT
+
+# Create config for client's csr
+cat > ${cert_conf_dir}/client.conf <<EOF
+[req]
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+prompt = no
+[req_distinguished_name]
+CN = kubernetes.invalid
+[req_ext]
+keyUsage=critical,digitalSignature,keyEncipherment
+extendedKeyUsage=clientAuth
+subjectAltName=dirName:kubelet,dirName:kubeproxy
+[kubelet]
+CN=kubelet
+[kubeproxy]
+CN=kube-proxy
+EOF
+
+# Generate client's private key and csr
+openssl genrsa -out "${CLIENT_KEY}" 4096
+openssl req -new -days 1000 \
+        -key "${CLIENT_KEY}" \
+        -out "${CLIENT_CSR}" \
+        -reqexts req_ext \
+        -config "${cert_conf_dir}/client.conf"
+
+# Send csr to Magnum to have it signed
+csr_req=$(python -c "import json; fp = open('${CLIENT_CSR}'); print json.dumps({'bay_uuid': '$BAY_UUID', 'csr': fp.read()}); fp.close()")
+curl -X POST \
+    -H "X-Auth-Token: $USER_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$csr_req" \
+    $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CLIENT_CERT}
+
+sed -i '
+s|CA_CERT|'"$CA_CERT"'|
+s|CLIENT_CERT|'"$CLIENT_CERT"'|
+s|CLIENT_KEY|'"$CLIENT_KEY"'|
+' /srv/kubernetes/kubeconfig.yaml

magnum/templates/heat-kubernetes/fragments/make-cert.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+. /etc/sysconfig/heat-params
+
+if [ "$TLS_DISABLED" == "True" ]; then
+  exit 0
+fi
+
+cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+sans="IP:${cert_ip},IP:${KUBE_API_PUBLIC_ADDRESS},IP:${KUBE_API_PRIVATE_ADDRESS},IP:127.0.0.1"
+MASTER_HOSTNAME=${MASTER_HOSTNAME:-}
+if [[ -n "${MASTER_HOSTNAME}" ]]; then
+  sans="${sans},DNS:${MASTER_HOSTNAME}"
+fi
+
+cert_dir=/srv/kubernetes
+cert_conf_dir=${cert_dir}/conf
+cert_group=root
+
+mkdir -p "$cert_dir"
+mkdir -p "$cert_conf_dir"
+
+CA_CERT=$cert_dir/ca.crt
+SERVER_CERT=$cert_dir/server.crt
+SERVER_CSR=$cert_dir/server.csr
+SERVER_KEY=$cert_dir/server.key
+
+# Get CA certificate for this bay
+curl -X GET \
+    -H "X-Auth-Token: $USER_TOKEN" \
+    $MAGNUM_URL/certificates/$BAY_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
+
+# Create config for server's csr
+cat > ${cert_conf_dir}/server.conf <<EOF
+[req]
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+prompt = no
+[req_distinguished_name]
+CN = kubernetes.invalid
+[req_ext]
+subjectAltName = ${sans}
+extendedKeyUsage = clientAuth,serverAuth
+EOF
+
+# Generate server's private key and csr
+openssl genrsa -out "${SERVER_KEY}" 4096
+openssl req -new -days 1000 \
+        -key "${SERVER_KEY}" \
+        -out "${SERVER_CSR}" \
+        -reqexts req_ext \
+        -config "${cert_conf_dir}/server.conf"
+
+# Send csr to Magnum to have it signed
+csr_req=$(python -c "import json; fp = open('${SERVER_CSR}'); print json.dumps({'bay_uuid': '$BAY_UUID', 'csr': fp.read()}); fp.close()")
+curl -X POST \
+    -H "X-Auth-Token: $USER_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$csr_req" \
+    $MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}

magnum/templates/heat-kubernetes/fragments/write-heat-params-master.yaml

@@ -5,6 +5,9 @@ write_files:
     owner: "root:root"
     permissions: "0644"
     content: |
+      KUBE_API_PUBLIC_ADDRESS="$KUBE_API_PUBLIC_ADDRESS"
+      KUBE_API_PRIVATE_ADDRESS="$KUBE_API_PRIVATE_ADDRESS"
+      KUBE_API_PORT="$KUBE_API_PORT"
       KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
       NETWORK_DRIVER="$NETWORK_DRIVER"
       FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
@@ -17,3 +20,7 @@ write_files:
       PASSWORD="$PASSWORD"
       TENANT_NAME="$TENANT_NAME"
       CLUSTER_SUBNET="$CLUSTER_SUBNET"
+      TLS_DISABLED="$TLS_DISABLED"
+      BAY_UUID="$BAY_UUID"
+      USER_TOKEN="$USER_TOKEN"
+      MAGNUM_URL="$MAGNUM_URL"

magnum/templates/heat-kubernetes/fragments/write-heat-params.yaml

@@ -7,6 +7,7 @@ write_files:
     content: |
       KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
       KUBE_MASTER_IP="$KUBE_MASTER_IP"
+      KUBE_API_PORT="$KUBE_API_PORT"
       ETCD_SERVER_IP="$ETCD_SERVER_IP"
       DOCKER_VOLUME="$DOCKER_VOLUME"
       NETWORK_DRIVER="$NETWORK_DRIVER"
@@ -21,3 +22,7 @@ write_files:
       REGISTRY_CONTAINER="$REGISTRY_CONTAINER"
       REGISTRY_INSECURE="$REGISTRY_INSECURE"
       REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE"
+      TLS_DISABLED="$TLS_DISABLED"
+      BAY_UUID="$BAY_UUID"
+      USER_TOKEN="$USER_TOKEN"
+      MAGNUM_URL="$MAGNUM_URL"

magnum/templates/heat-kubernetes/fragments/write-kubeconfig.yaml

@@ -0,0 +1,24 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /srv/kubernetes/kubeconfig.yaml
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      apiVersion: v1
+      kind: Config
+      users:
+      - name: kubeclient
+        user:
+          client-certificate: CLIENT_CERT
+          client-key: CLIENT_KEY
+      clusters:
+      - name: kubernetes
+        cluster:
+          certificate-authority: CA_CERT
+      contexts:
+      - context:
+          cluster: kubernetes
+          user: kubeclient
+        name: service-account-context
+      current-context: service-account-context

magnum/templates/heat-kubernetes/kubecluster.yaml

@@ -188,7 +188,7 @@ parameters:
   password:
     type: string
     description: >
-      user password, not set in current implementation, only used to 
+      user password, not set in current implementation, only used to
       fill in for Kubernetes config file
     default:
       ChangeMe
@@ -198,6 +198,39 @@ parameters:
     description: >
       tenant name
 
+  loadbalancing_protocol:
+    type: string
+    description: >
+      The protocol which is used for load balancing. If you want to change
+      tls_disabled option to 'True', please change this to "HTTP".
+    default: TCP
+    constraints:
+      - allowed_values: ["TCP", "HTTP"]
+
+  tls_disabled:
+    type: boolean
+    description: whether or not to disable TLS
+    default: False
+
+  kubernetes_port:
+    type: number
+    description: >
+      The port which are used by kube-apiserver to provide Kubernetes
+      service.
+    default: 6443
+
+  user_token:
+    type: string
+    description: token used for communicating back to Magnum for TLS certs
+
+  bay_uuid:
+    type: string
+    description: identifier for the bay this template is generating
+
+  magnum_url:
+    type: string
+    description: endpoint to retrieve TLS certs from
+
 resources:
 
   ######################################################################
@@ -249,12 +282,12 @@ resources:
   api_pool:
     type: OS::Neutron::Pool
     properties:
-      protocol: HTTP
+      protocol: {get_param: loadbalancing_protocol}
       monitors: [{get_resource: api_monitor}]
       subnet: {get_resource: fixed_subnet}
       lb_method: ROUND_ROBIN
       vip:
-        protocol_port: 8080
+        protocol_port: {get_param: kubernetes_port}
 
   api_pool_floating:
     type: OS::Neutron::FloatingIP
@@ -295,6 +328,8 @@ resources:
       resource_def:
         type: kubemaster.yaml
         properties:
+          api_public_address: {get_attr: [api_pool_floating, floating_ip_address]}
+          api_private_address: {get_attr: [api_pool, vip, address]}
           ssh_key_name: {get_param: ssh_key_name}
           server_image: {get_param: server_image}
           master_flavor: {get_param: master_flavor}
@@ -307,6 +342,9 @@ resources:
           flannel_use_vxlan: {get_param: flannel_use_vxlan}
           portal_network_cidr: {get_param: portal_network_cidr}
           discovery_url: {get_param: discovery_url}
+          user_token: {get_param: user_token}
+          bay_uuid: {get_param: bay_uuid}
+          magnum_url: {get_param: magnum_url}
           fixed_network: {get_resource: fixed_network}
           fixed_subnet: {get_resource: fixed_subnet}
           api_pool_id: {get_resource: api_pool}
@@ -315,6 +353,8 @@ resources:
           username: {get_param: username}
           password: {get_param: password}
           tenant_name: {get_param: tenant_name}
+          kubernetes_port: {get_param: kubernetes_port}
+          tls_disabled: {get_param: tls_disabled}
 
   ######################################################################
   #
@@ -356,13 +396,18 @@ resources:
           registry_container: {get_param: registry_container}
           registry_insecure: {get_param: registry_insecure}
           registry_chunksize: {get_param: registry_chunksize}
+          user_token: {get_param: user_token}
+          bay_uuid: {get_param: bay_uuid}
+          magnum_url: {get_param: magnum_url}
+          kubernetes_port: {get_param: kubernetes_port}
+          tls_disabled: {get_param: tls_disabled}
 
 outputs:
 
   api_address:
     value:
       str_replace:
-        template: api_ip_address:8080
+        template: api_ip_address
         params:
           api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]}
     description: >

magnum/templates/heat-kubernetes/kubemaster.yaml

@@ -63,7 +63,37 @@ parameters:
     description: >
       Discovery URL used for bootstrapping the etcd cluster.
 
+  tls_disabled:
+    type: boolean
+    description: whether or not to enable TLS
+    default: False
+
+  kubernetes_port:
+    type: number
+    description: >
+      The port which are used by kube-apiserver to provide Kubernetes
+      service.
+    default: 6443
+
+  user_token:
+    type: string
+    description: token used for communicating back to Magnum for TLS certs
+
+  bay_uuid:
+    type: string
+    description: identifier for the bay this template is generating
+
+  magnum_url:
+    type: string
+    description: endpoint to retrieve TLS certs from
+
   # The following are all generated in the parent template.
+  api_public_address:
+    type: string
+    description: Public IP address of the Kubernetes master server.
+  api_private_address:
+    type: string
+    description: Private IP address of the Kubernetes master server.
   fixed_network:
     type: string
     description: Network from which to allocate fixed addresses.
@@ -135,8 +165,8 @@ resources:
           port_range_min: 7080
           port_range_max: 7080
         - protocol: tcp
-          port_range_min: 8080
+          port_range_min: {get_param: kubernetes_port}
-          port_range_max: 8080
+          port_range_max: {get_param: kubernetes_port}
         - protocol: tcp
           port_range_min: 2379
           port_range_max: 2379
@@ -158,6 +188,9 @@ resources:
         str_replace:
           template: {get_file: fragments/write-heat-params-master.yaml}
           params:
+            "$KUBE_API_PUBLIC_ADDRESS": {get_param: api_public_address}
+            "$KUBE_API_PRIVATE_ADDRESS": {get_param: api_private_address}
+            "$KUBE_API_PORT": {get_param: kubernetes_port}
             "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
             "$NETWORK_DRIVER": {get_param: network_driver}
             "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
@@ -170,6 +203,16 @@ resources:
             "$PASSWORD": {get_param: password}
             "$TENANT_NAME": {get_param: tenant_name}
             "$CLUSTER_SUBNET": {get_param: fixed_subnet}
+            "$TLS_DISABLED": {get_param: tls_disabled}
+            "$BAY_UUID": {get_param: bay_uuid}
+            "$USER_TOKEN": {get_param: user_token}
+            "$MAGNUM_URL": {get_param: magnum_url}
+
+  make_cert:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/make-cert.sh}
 
   configure_etcd:
     type: OS::Heat::SoftwareConfig
@@ -246,6 +289,7 @@ resources:
         - config: {get_resource: configure_etcd}
         - config: {get_resource: kube_user}
         - config: {get_resource: write_kube_os_config}
+        - config: {get_resource: make_cert}
         - config: {get_resource: configure_kubernetes}
         - config: {get_resource: enable_services}
         - config: {get_resource: write_network_config}
@@ -291,7 +335,7 @@ resources:
     properties:
       pool_id: {get_param: api_pool_id}
       address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
-      protocol_port: 8080
+      protocol_port: {get_param: kubernetes_port}
 
   etcd_pool_member:
     type: OS::Neutron::PoolMember

magnum/templates/heat-kubernetes/kubeminion.yaml

@@ -40,6 +40,31 @@ parameters:
       storage
     default: 25
 
+  tls_disabled:
+    type: boolean
+    description: whether or not to enable TLS
+    default: False
+
+  kubernetes_port:
+    type: number
+    description: >
+      The port which are used by kube-apiserver to provide Kubernetes
+      service.
+    default: 6443
+
+  user_token:
+    type: string
+    description: token used for communicating back to Magnum for TLS certs
+
+  bay_uuid:
+    type: string
+    description: identifier for the bay this template is generating
+
+  magnum_url:
+    type: string
+    description: endpoint to retrieve TLS certs from
+
+
   # The following are all generated in the parent template.
   kube_master_ip:
     type: string
@@ -156,6 +181,7 @@ resources:
           params:
             $KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
             $KUBE_MASTER_IP: {get_param: kube_master_ip}
+            $KUBE_API_PORT: {get_param: kubernetes_port}
             $ETCD_SERVER_IP: {get_param: etcd_server_ip}
             $DOCKER_VOLUME: {get_resource: docker_volume}
             $NETWORK_DRIVER: {get_param: network_driver}
@@ -170,6 +196,22 @@ resources:
             $REGISTRY_CONTAINER: {get_param: registry_container}
             $REGISTRY_INSECURE: {get_param: registry_insecure}
             $REGISTRY_CHUNKSIZE: {get_param: registry_chunksize}
+            $TLS_DISABLED: {get_param: tls_disabled}
+            $BAY_UUID: {get_param: bay_uuid}
+            $USER_TOKEN: {get_param: user_token}
+            $MAGNUM_URL: {get_param: magnum_url}
+
+  write_kubeconfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/write-kubeconfig.yaml}
+
+  make_cert:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/make-cert-client.sh}
 
   configure_docker_storage:
     type: OS::Heat::SoftwareConfig
@@ -250,6 +292,8 @@ resources:
         - config: {get_resource: disable_selinux}
         - config: {get_resource: write_heat_params}
         - config: {get_resource: kube_user}
+        - config: {get_resource: write_kubeconfig}
+        - config: {get_resource: make_cert}
         - config: {get_resource: kube_examples}
         - config: {get_resource: configure_docker_storage}
         - config: {get_resource: kube_register}

magnum/tests/functional/python_client_base.py

@@ -103,6 +103,9 @@ class BaseMagnumClient(base.TestCase):
             network_driver='flannel',
             coe=coe,
             labels={"K1": "V1", "K2": "V2"},
+            # TODO(yuanying): Change to `tls_disabled=False`
+            #                 if k8sclient supports TLS.
+            tls_disabled=True,
         )
         return baymodel
 

magnum/tests/functional/test_k8s_python_client.py

@@ -38,8 +38,7 @@ class TestKubernetesAPIs(BaseMagnumClient):
         super(TestKubernetesAPIs, cls).setUpClass()
         cls.baymodel = cls._create_baymodel('testk8sAPI')
         cls.bay = cls._create_bay('testk8sAPI', cls.baymodel.uuid)
-        kube_api_address = cls.cs.bays.get(cls.bay.uuid).api_address
+        kube_api_url = cls.cs.bays.get(cls.bay.uuid).api_address
-        kube_api_url = 'http://%s' % kube_api_address
         k8s_client = api_client.ApiClient(kube_api_url)
         cls.k8s_api = apiv_api.ApivApi(k8s_client)
 

magnum/tests/unit/conductor/handlers/test_bay_conductor.py

@@ -52,9 +52,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'labels': {'flannel_network_cidr': '10.101.0.0/16',
                        'flannel_network_subnetlen': '26',
                        'flannel_use_vxlan': 'yes'},
-
+            'tls_disabled': False,
         }
         self.bay_dict = {
+            'uuid': 'bay-xx-xx-xx-xx',
             'baymodel_id': 'xx-xx-xx-xx',
             'name': 'bay1',
             'stack_id': 'xx-xx-xx-xx',
@@ -70,6 +71,12 @@ class TestBayConductorWithK8s(base.TestCase):
         self.context.auth_url = 'http://192.168.10.10:5000/v3'
         self.context.user_name = 'fake_user'
         self.context.tenant = 'fake_tenant'
+        osc_patcher = mock.patch('magnum.common.clients.OpenStackClients')
+        self.mock_osc_class = osc_patcher.start()
+        self.addCleanup(osc_patcher.stop)
+        self.mock_osc = mock.MagicMock()
+        self.mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
+        self.mock_osc_class.return_value = self.mock_osc
 
     @patch('magnum.objects.BayModel.get_by_uuid')
     def test_extract_template_definition(
@@ -112,6 +119,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'http_proxy': 'http_proxy',
             'https_proxy': 'https_proxy',
             'no_proxy': 'no_proxy',
+            'user_token': self.context.auth_token,
+            'bay_uuid': self.bay_dict['uuid'],
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
         }
         expected = {
             'ssh_key_name': 'keypair_id',
@@ -135,6 +146,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v2',
             'tenant_name': 'fake_tenant',
             'username': 'fake_user',
+            'user_token': self.context.auth_token,
+            'bay_uuid': self.bay_dict['uuid'],
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
         }
         if missing_attr is not None:
             expected.pop(mapping[missing_attr], None)
@@ -186,6 +201,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v2',
             'tenant_name': 'fake_tenant',
             'username': 'fake_user',
+            'user_token': self.context.auth_token,
+            'bay_uuid': self.bay_dict['uuid'],
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
         }
         self.assertEqual(expected, definition)
 
@@ -234,6 +253,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v2',
             'tenant_name': 'fake_tenant',
             'username': 'fake_user',
+            'user_token': self.context.auth_token,
+            'bay_uuid': self.bay_dict['uuid'],
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
         }
         self.assertEqual(expected, definition)
 
@@ -322,6 +345,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v2',
             'tenant_name': 'fake_tenant',
             'username': 'fake_user',
+            'user_token': self.context.auth_token,
+            'bay_uuid': self.bay_dict['uuid'],
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
         }
         self.assertIn('token', definition)
         del definition['token']
@@ -395,6 +422,10 @@ class TestBayConductorWithK8s(base.TestCase):
             'auth_url': 'http://192.168.10.10:5000/v2',
             'tenant_name': 'fake_tenant',
             'username': 'fake_user',
+            'user_token': self.context.auth_token,
+            'bay_uuid': self.bay_dict['uuid'],
+            'magnum_url': self.mock_osc.magnum_url.return_value,
+            'tls_disabled': False,
         }
         self.assertEqual(expected, definition)
         reqget.assert_called_once_with('http://etcd/test?size=1')

magnum/tests/unit/conductor/handlers/test_k8s_conductor.py

@@ -12,8 +12,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from oslo_config import cfg
-
 from magnum.common import exception
 from magnum.common.pythonk8sclient.swagger_client import rest
 from magnum.conductor.handlers import k8s_conductor
@@ -24,12 +22,6 @@ import mock
 from mock import patch
 
 
-cfg.CONF.import_opt('k8s_protocol', 'magnum.conductor.handlers.k8s_conductor',
-                    group='kubernetes')
-cfg.CONF.import_opt('k8s_port', 'magnum.conductor.handlers.k8s_conductor',
-                    group='kubernetes')
-
-
 class TestK8sConductor(base.TestCase):
     def setUp(self):
         super(TestK8sConductor, self).setUp()

magnum/tests/unit/conductor/test_k8s_api.py

@@ -13,7 +13,6 @@
 # under the License.
 
 from mock import patch
-from oslo_config import cfg
 
 from magnum.conductor import k8s_api
 from magnum import objects
@@ -26,7 +25,6 @@ class TestK8sAPI(base.TestCase):
     def test_retrieve_k8s_api_endpoint(self, mock_bay_get_by_uuid):
         expected_context = 'context'
         expected_api_address = 'api_address'
-        expected_protocol = cfg.CONF.kubernetes.k8s_protocol
 
         resource = objects.Pod({})
         resource.bay_uuid = 'bay_uuid'
@@ -37,9 +35,7 @@ class TestK8sAPI(base.TestCase):
 
         actual_api_endpoint = k8s_api.K8sAPI._retrieve_k8s_api_endpoint(
             expected_context, resource)
-        self.assertEqual("%s://%s" % (expected_protocol,
+        self.assertEqual(expected_api_address, actual_api_endpoint)
-                                      expected_api_address),
-                         actual_api_endpoint)
 
     @patch('magnum.conductor.k8s_api.K8sAPI')
     def test_create_k8s_api(self, mock_k8s_api_cls):

magnum/tests/unit/conductor/test_template_definition.py

@@ -136,22 +136,35 @@ class TemplateDefinitionTestCase(base.TestCase):
         value = output.get_output_value(mock_stack)
         self.assertIsNone(value)
 
+    def test_add_output_with_mapping_type(self):
+        definition = tdef.TemplateDefinition.get_template_definition(
+            'vm',
+            'fedora-atomic',
+            'kubernetes')
+
+        mock_args = [1, 3, 4]
+        mock_kwargs = {'test': 'test'}
+        mock_mapping_type = mock.MagicMock()
+        mock_mapping_type.return_value = mock.MagicMock()
+        definition.add_output(mapping_type=mock_mapping_type, *mock_args,
+                              **mock_kwargs)
+
+        mock_mapping_type.assert_called_once_with(*mock_args, **mock_kwargs)
+        self.assertIn(mock_mapping_type.return_value,
+                      definition.output_mappings)
+
     def test_update_outputs(self):
         definition = tdef.TemplateDefinition.get_template_definition(
             'vm',
             'fedora-atomic',
             'kubernetes')
 
-        expected_api_address = 'api_address'
         expected_node_addresses = ['ex_minion', 'address']
 
         outputs = [
             {"output_value": expected_node_addresses,
              "description": "No description given",
              "output_key": "kube_minions_external"},
-            {"output_value": expected_api_address,
-             "description": "No description given",
-             "output_key": "api_address"},
             {"output_value": ['any', 'output'],
              "description": "No description given",
              "output_key": "kube_minions"}
@@ -159,15 +172,17 @@ class TemplateDefinitionTestCase(base.TestCase):
         mock_stack = mock.MagicMock()
         mock_stack.outputs = outputs
         mock_bay = mock.MagicMock()
+        mock_bay.api_address = None
+        mock_baymodel = mock.MagicMock()
 
-        definition.update_outputs(mock_stack, mock_bay)
+        definition.update_outputs(mock_stack, mock_baymodel, mock_bay)
 
-        self.assertEqual(mock_bay.api_address, expected_api_address)
         self.assertEqual(mock_bay.node_addresses, expected_node_addresses)
 
 
 class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
 
+    @mock.patch('magnum.common.clients.OpenStackClients')
     @mock.patch('magnum.conductor.template_definition'
                 '.AtomicK8sTemplateDefinition.get_discovery_url')
     @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition'
@@ -175,11 +190,17 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
     @mock.patch('magnum.conductor.template_definition.TemplateDefinition'
                 '.get_output')
     def test_k8s_get_params(self, mock_get_output, mock_get_params,
-                            mock_get_discovery_url):
+                            mock_get_discovery_url, mock_osc_class):
         mock_context = mock.MagicMock()
+        mock_context.auth_token = 'AUTH_TOKEN'
         mock_baymodel = mock.MagicMock()
+        mock_baymodel.tls_disabled = False
         mock_bay = mock.MagicMock()
+        mock_bay.uuid = 'bay-xx-xx-xx-xx'
         mock_scale_manager = mock.MagicMock()
+        mock_osc = mock.MagicMock()
+        mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
+        mock_osc_class.return_value = mock_osc
 
         removal_nodes = ['node1', 'node2']
         mock_scale_manager.get_removal_nodes.return_value = removal_nodes
@@ -206,7 +227,62 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
             'flannel_network_subnetlen': flannel_vxlan,
             'auth_url': 'http://192.168.10.10:5000/v2',
             'username': 'fake_user',
-            'tenant_name': 'fake_tenant'}}
+            'tenant_name': 'fake_tenant',
+            'magnum_url': mock_osc.magnum_url.return_value,
+            'user_token': mock_context.auth_token}}
+        mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
+                                                mock_bay, **expected_kwargs)
+
+    @mock.patch('magnum.common.clients.OpenStackClients')
+    @mock.patch('magnum.conductor.template_definition'
+                '.AtomicK8sTemplateDefinition.get_discovery_url')
+    @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition'
+                '.get_params')
+    @mock.patch('magnum.conductor.template_definition.TemplateDefinition'
+                '.get_output')
+    def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params,
+                                     mock_get_discovery_url, mock_osc_class):
+        mock_context = mock.MagicMock()
+        mock_context.auth_token = 'AUTH_TOKEN'
+        mock_baymodel = mock.MagicMock()
+        mock_baymodel.tls_disabled = True
+        mock_bay = mock.MagicMock()
+        mock_bay.uuid = 'bay-xx-xx-xx-xx'
+        mock_scale_manager = mock.MagicMock()
+        mock_osc = mock.MagicMock()
+        mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
+        mock_osc_class.return_value = mock_osc
+
+        removal_nodes = ['node1', 'node2']
+        mock_scale_manager.get_removal_nodes.return_value = removal_nodes
+        mock_get_discovery_url.return_value = 'fake_discovery_url'
+
+        mock_context.auth_url = 'http://192.168.10.10:5000/v3'
+        mock_context.user_name = 'fake_user'
+        mock_context.tenant = 'fake_tenant'
+
+        flannel_cidr = mock_baymodel.labels.get('flannel_network_cidr')
+        flannel_subnet = mock_baymodel.labels.get('flannel_network_subnetlen')
+        flannel_vxlan = mock_baymodel.labels.get('flannel_use_vxlan')
+
+        k8s_def = tdef.AtomicK8sTemplateDefinition()
+
+        k8s_def.get_params(mock_context, mock_baymodel, mock_bay,
+                           scale_manager=mock_scale_manager)
+
+        expected_kwargs = {'extra_params': {
+            'minions_to_remove': removal_nodes,
+            'discovery_url': 'fake_discovery_url',
+            'flannel_network_cidr': flannel_cidr,
+            'flannel_use_vxlan': flannel_subnet,
+            'flannel_network_subnetlen': flannel_vxlan,
+            'auth_url': 'http://192.168.10.10:5000/v2',
+            'username': 'fake_user',
+            'tenant_name': 'fake_tenant',
+            'magnum_url': mock_osc.magnum_url.return_value,
+            'user_token': mock_context.auth_token,
+            'loadbalancing_protocol': 'HTTP',
+            'kubernetes_port': 8080}}
         mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
                                                 mock_bay, **expected_kwargs)
 
@@ -249,6 +325,67 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
                           tdef.AtomicK8sTemplateDefinition().get_discovery_url,
                           fake_bay)
 
+    def test_update_outputs_api_address(self):
+        definition = tdef.TemplateDefinition.get_template_definition(
+            'vm',
+            'fedora-atomic',
+            'kubernetes')
+
+        address = 'updated_address'
+        protocol = 'http'
+        port = '8080'
+        params = {
+            'protocol': protocol,
+            'address': address,
+            'port': port,
+        }
+        expected_api_address = '%(protocol)s://%(address)s:%(port)s' % params
+
+        outputs = [
+            {"output_value": address,
+             "description": "No description given",
+             "output_key": "api_address"},
+        ]
+        mock_stack = mock.MagicMock()
+        mock_stack.outputs = outputs
+        mock_bay = mock.MagicMock()
+        mock_baymodel = mock.MagicMock()
+        mock_baymodel.tls_disabled = True
+
+        definition.update_outputs(mock_stack, mock_baymodel, mock_bay)
+
+        self.assertEqual(mock_bay.api_address, expected_api_address)
+
+    def test_update_outputs_if_baymodel_is_secure(self):
+        definition = tdef.TemplateDefinition.get_template_definition(
+            'vm',
+            'fedora-atomic',
+            'kubernetes')
+
+        address = 'updated_address'
+        protocol = 'https'
+        port = '6443'
+        params = {
+            'protocol': protocol,
+            'address': address,
+            'port': port,
+        }
+        expected_api_address = '%(protocol)s://%(address)s:%(port)s' % params
+
+        outputs = [
+            {"output_value": address,
+             "description": "No description given",
+             "output_key": "api_address"},
+        ]
+        mock_stack = mock.MagicMock()
+        mock_stack.outputs = outputs
+        mock_bay = mock.MagicMock()
+        mock_baymodel = mock.MagicMock()
+        mock_baymodel.tls_disabled = False
+
+        definition.update_outputs(mock_stack, mock_baymodel, mock_bay)
+        self.assertEqual(mock_bay.api_address, expected_api_address)
+
 
 class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):