[k8s] allow enabling kubernetes cert manager api

Add a new label 'cert_manager_api' to kubernetes clusters controlling the enable/disable of the kubernetes certificate manager api. The same cluster cert/key pair is used by this api. The heat agent is used to install the key in the master node(s), as this is required for kubernetes to later sign new certificate requests. The master template init order is changed so the heat agent is launched previous to enabling the services - the controller manager requires the CA key to be locally available before being launched. Change-Id: Ibf85147316e3a194d8a3f92cbb4ae9ce8e16c98f Partial-Bug: #1734318
2017-12-22 11:07:51 +00:00 · 2017-12-22 11:07:51 +00:00 · faa9e90402
commit faa9e90402
parent c6e7b290ab
12 changed files with 119 additions and 13 deletions
--- a/doc/source/user/index.rst
+++ b/doc/source/user/index.rst
@ -348,10 +348,11 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `container_infra_prefix`_             | see below          | ""            |
 +---------------------------------------+--------------------+---------------+
-+---------------------------------------+--------------------+---------------+
 | `availability_zone`_                  | AZ for the cluster | ""            |
 |                                       | nodes              |               |
 +---------------------------------------+--------------------+---------------+
+| `cert_manager_api`_                   | see below          | false         |
+---------------------------------------+--------------------+---------------+

 Cluster
 -------
@ -1113,6 +1114,10 @@ _`kube_dashboard_enabled`
  This label triggers the deployment of the kubernetes dashboard.
  The default value is 1, meaning it will be enabled.

+_`cert_manager_api`
+  This label enables the kubernetes `certificate manager api
+  <https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/>`_.
+
 External load balancer for services
 -----------------------------------

--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@ -57,6 +57,10 @@ if [ -n "$TRUST_ID" ]; then
    KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/kubernetes/kube_openstack_config --cloud-provider=openstack"
 fi

+if [ -n "$CERT_MANAGER_API" ]; then
+    KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-signing-cert-file=$CERT_DIR/ca.crt --cluster-signing-key-file=$CERT_DIR/ca.key"
+fi
+
 sed -i '
    /^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/
    /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager
@ -0,0 +1,15 @@
+#!/bin/bash
+
+. /etc/sysconfig/heat-params
+
+if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    exit 0
+fi
+
+cert_dir=/etc/kubernetes/certs
+
+echo -e "$CA_KEY" > ${cert_dir}/ca.key
+
+chown kube.kube ${cert_dir}/ca.key
+chmod 400 ${cert_dir}/ca.key
+
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
@ -1,8 +1,18 @@
 #!/bin/sh

+. /etc/sysconfig/heat-params
+
 # make sure we pick up any modified unit files
 systemctl daemon-reload

+# if the certificate manager api is enabled, wait for the ca key to be handled
+# by the heat container agent (required for the controller-manager)
+while [ ! -f /etc/kubernetes/certs/ca.key ] && \
+    [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" == "true" ]; do
+    echo "waiting for CA to be made available for certificate manager api"
+    sleep 2
+done
+
 echo "starting services"
 for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler; do
    echo "activating service $service"
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
@ -52,3 +52,5 @@ write_files:
      ETCD_LB_VIP="$ETCD_LB_VIP"
      DNS_SERVICE_IP="$DNS_SERVICE_IP"
      DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
+      CERT_MANAGER_API="$CERT_MANAGER_API"
+      CA_KEY="$CA_KEY"
--- a/magnum/drivers/heat/k8s_fedora_template_def.py
+++ b/magnum/drivers/heat/k8s_fedora_template_def.py
@ -11,7 +11,10 @@
 # under the License.

 from oslo_log import log as logging
+from oslo_utils import strutils

+from magnum.common.x509 import operations as x509
+from magnum.conductor.handlers.common import cert_manager
 from magnum.drivers.heat import k8s_template_def
 from magnum.drivers.heat import template_def
 from oslo_config import cfg
@ -88,6 +91,14 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
            if label_value:
                extra_params[label] = label_value

+        cert_manager_api = cluster.labels.get('cert_manager_api')
+        if strutils.bool_from_string(cert_manager_api):
+            extra_params['cert_manager_api'] = cert_manager_api
+            ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
+            extra_params['ca_key'] = x509.decrypt_key(
+                ca_cert.get_private_key(),
+                ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
+
        return super(K8sFedoraTemplateDefinition,
                     self).get_params(context, cluster_template, cluster,
                                      extra_params=extra_params,
--- a/magnum/drivers/heat/k8s_template_def.py
+++ b/magnum/drivers/heat/k8s_template_def.py
@ -113,7 +113,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
                      'prometheus_monitoring',
                      'grafana_admin_passwd',
                      'kube_dashboard_enabled',
-                      'etcd_volume_size']
+                      'etcd_volume_size',
+                      'cert_manager_api']

        for label in label_list:
            extra_params[label] = cluster.labels.get(label)
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
@ -370,6 +370,17 @@ parameters:
      availability zone for master and nodes
    default: ""

+  cert_manager_api:
+    type: boolean
+    description: true if the kubernetes cert api manager should be enabled
+    default: false
+
+  ca_key:
+    type: string
+    description: key of internal ca for the kube certificate api manager
+    default: ""
+    hidden: true
+
 resources:

  ######################################################################
@ -564,6 +575,8 @@ resources:
          openstack_ca: {get_param: openstack_ca}
          nodes_server_group_id: {get_resource: nodes_server_group}
          availability_zone: {get_param: availability_zone}
+          ca_key: {get_param: ca_key}
+          cert_manager_api: {get_param: cert_manager_api}

  ######################################################################
  #
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
@ -273,6 +273,16 @@ parameters:
      availability zone for master and nodes
    default: ""

+  ca_key:
+    type: string
+    description: key of internal ca for the kube certificate api manager
+    hidden: true
+
+  cert_manager_api:
+    type: boolean
+    description: true if the kubernetes cert api manager should be enabled
+    default: false
+
 resources:

  master_wait_handle:
@ -359,6 +369,8 @@ resources:
            "$ETCD_LB_VIP": {get_param: etcd_lb_vip}
            "$DNS_SERVICE_IP": {get_param: dns_service_ip}
            "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
+            "$CERT_MANAGER_API": {get_param: cert_manager_api}
+            "$CA_KEY": {get_param: ca_key}

  install_openstack_ca:
    type: OS::Heat::SoftwareConfig
@ -483,6 +495,7 @@ resources:
        - config: {get_resource: configure_docker_storage}
        - config: {get_resource: configure_kubernetes}
        - config: {get_resource: add_proxy}
+        - config: {get_resource: start_container_agent}
        - config: {get_resource: enable_services}
        - config: {get_resource: write_network_config}
        - config: {get_resource: network_config_service}
@ -490,7 +503,6 @@ resources:
        - config: {get_resource: kube_apiserver_to_kubelet_role}
        - config: {get_resource: core_dns_service}
        - config: {get_resource: kube_ui_service}
-        - config: {get_resource: start_container_agent}
        - config: {get_resource: master_wc_notify}

  enable_prometheus_monitoring:
@ -511,6 +523,24 @@ resources:
      server: {get_resource: kube-master}
      actions: ['CREATE']

+  enable_cert_manager_api:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        str_replace:
+          template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager}
+          params:
+            "$CA_KEY": {get_param: ca_key}
+
+  enable_cert_manager_api_deployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      signal_transport: HEAT_SIGNAL
+      config: {get_resource: enable_cert_manager_api}
+      server: {get_resource: kube-master}
+      actions: ['CREATE']
+
  ######################################################################
  #
  # a single kubernetes master.
--- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
@ -97,7 +97,8 @@ class TestClusterConductorWithK8s(base.TestCase):
                       'grafana_admin_passwd': 'fake_pwd',
                       'kube_dashboard_enabled': 'True',
                       'docker_volume_type': 'lvmdriver-1',
-                       'availability_zone': 'az_1'},
+                       'availability_zone': 'az_1',
+                       'cert_manager_api': 'False'},
            'master_flavor_id': 'master_flavor_id',
            'flavor_id': 'flavor_id',
        }
@ -177,7 +178,8 @@ class TestClusterConductorWithK8s(base.TestCase):
                       'kube_dashboard_enabled': 'True',
                       'docker_volume_type': 'lvmdriver-1',
                       'etcd_volume_size': None,
-                       'availability_zone': 'az_1'},
+                       'availability_zone': 'az_1',
+                       'cert_manager_api': 'False'},
            'http_proxy': 'http_proxy',
            'https_proxy': 'https_proxy',
            'no_proxy': 'no_proxy',
@ -233,7 +235,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'verify_ca': True,
            'openstack_ca': '',
            "nodes_affinity_policy": "soft-anti-affinity",
-            'availability_zone': 'az_1'
+            'availability_zone': 'az_1',
+            'cert_manager_api': 'False',
        }
        if missing_attr is not None:
            expected.pop(mapping[missing_attr], None)
@ -331,7 +334,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'verify_ca': True,
            'openstack_ca': '',
            "nodes_affinity_policy": "soft-anti-affinity",
-            'availability_zone': 'az_1'
+            'availability_zone': 'az_1',
+            'cert_manager_api': 'False',
        }

        self.assertEqual(expected, definition)
@ -416,7 +420,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'verify_ca': True,
            'openstack_ca': '',
            "nodes_affinity_policy": "soft-anti-affinity",
-            'availability_zone': 'az_1'
+            'availability_zone': 'az_1',
+            'cert_manager_api': 'False',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
@ -494,7 +499,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'insecure_registry_url': '10.0.0.1:5000',
            'kube_version': 'fake-version',
            'verify_ca': True,
-            'openstack_ca': ''
+            'openstack_ca': '',
+            'cert_manager_api': 'False',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
@ -567,7 +573,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'insecure_registry_url': '10.0.0.1:5000',
            'kube_version': 'fake-version',
            'verify_ca': True,
-            'openstack_ca': ''
+            'openstack_ca': '',
+            'cert_manager_api': 'False',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
@ -741,7 +748,8 @@ class TestClusterConductorWithK8s(base.TestCase):
            'verify_ca': True,
            'openstack_ca': '',
            "nodes_affinity_policy": "soft-anti-affinity",
-            'availability_zone': 'az_1'
+            'availability_zone': 'az_1',
+            'cert_manager_api': 'False',
        }
        self.assertEqual(expected, definition)
        self.assertEqual(
--- a/magnum/tests/unit/drivers/test_template_definition.py
+++ b/magnum/tests/unit/drivers/test_template_definition.py
@ -274,6 +274,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'container_infra_prefix')
        availability_zone = mock_cluster.labels.get(
            'availability_zone')
+        cert_manager_api = mock_cluster.labels.get('cert_manager_api')

        k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()

@ -301,7 +302,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'container_infra_prefix': container_infra_prefix,
            'nodes_affinity_policy': 'soft-anti-affinity',
            'availability_zone': availability_zone,
-        }}
+            'cert_manager_api': cert_manager_api}}
        mock_get_params.assert_called_once_with(mock_context,
                                                mock_cluster_template,
                                                mock_cluster,
@ -362,6 +363,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'container_infra_prefix')
        availability_zone = mock_cluster.labels.get(
            'availability_zone')
+        cert_manager_api = mock_cluster.labels.get('cert_manager_api')

        k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()

@ -391,7 +393,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
            'container_infra_prefix': container_infra_prefix,
            'nodes_affinity_policy': 'soft-anti-affinity',
            'availability_zone': availability_zone,
-        }}
+            'cert_manager_api': cert_manager_api}}
        mock_get_params.assert_called_once_with(mock_context,
                                                mock_cluster_template,
                                                mock_cluster,
--- a/releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml
+++ b/releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml
@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add new label 'cert_manager_api' enabling the kubernetes certificate
+    manager api.