From faa9e90402bcf78acdd166198fff9612fa8be81c Mon Sep 17 00:00:00 2001 From: Ricardo Rocha Date: Fri, 22 Dec 2017 11:07:51 +0000 Subject: [PATCH] [k8s] allow enabling kubernetes cert manager api Add a new label 'cert_manager_api' to kubernetes clusters controlling the enable/disable of the kubernetes certificate manager api. The same cluster cert/key pair is used by this api. The heat agent is used to install the key in the master node(s), as this is required for kubernetes to later sign new certificate requests. The master template init order is changed so the heat agent is launched previous to enabling the services - the controller manager requires the CA key to be locally available before being launched. Change-Id: Ibf85147316e3a194d8a3f92cbb4ae9ce8e16c98f Partial-Bug: #1734318 --- doc/source/user/index.rst | 7 +++- .../fragments/configure-kubernetes-master.sh | 4 +++ .../fragments/enable-cert-api-manager | 15 +++++++++ .../fragments/enable-services-master.sh | 10 ++++++ .../fragments/write-heat-params-master.yaml | 2 ++ .../drivers/heat/k8s_fedora_template_def.py | 11 +++++++ magnum/drivers/heat/k8s_template_def.py | 3 +- .../templates/kubecluster.yaml | 13 ++++++++ .../templates/kubemaster.yaml | 32 ++++++++++++++++++- .../handlers/test_k8s_cluster_conductor.py | 24 +++++++++----- .../unit/drivers/test_template_definition.py | 6 ++-- .../cert-manager-api-ee0cf7f3b767bb5d.yaml | 5 +++ 12 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager create mode 100644 releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index a4a99afa3b..9b29ad6e02 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -348,10 +348,11 @@ the table are linked to more details elsewhere in the user guide. +---------------------------------------+--------------------+---------------+ | `container_infra_prefix`_ | see below | "" | +---------------------------------------+--------------------+---------------+ -+---------------------------------------+--------------------+---------------+ | `availability_zone`_ | AZ for the cluster | "" | | | nodes | | +---------------------------------------+--------------------+---------------+ +| `cert_manager_api`_ | see below | false | ++---------------------------------------+--------------------+---------------+ Cluster ------- @@ -1113,6 +1114,10 @@ _`kube_dashboard_enabled` This label triggers the deployment of the kubernetes dashboard. The default value is 1, meaning it will be enabled. +_`cert_manager_api` + This label enables the kubernetes `certificate manager api + `_. + External load balancer for services ----------------------------------- diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index fa241e30ab..1cd5298aaa 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -57,6 +57,10 @@ if [ -n "$TRUST_ID" ]; then KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/kubernetes/kube_openstack_config --cloud-provider=openstack" fi +if [ -n "$CERT_MANAGER_API" ]; then + KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-signing-cert-file=$CERT_DIR/ca.crt --cluster-signing-key-file=$CERT_DIR/ca.key" +fi + sed -i ' /^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/ /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"# diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager b/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager new file mode 100644 index 0000000000..8a96baeceb --- /dev/null +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager @@ -0,0 +1,15 @@ +#!/bin/bash + +. /etc/sysconfig/heat-params + +if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" = "false" ]; then + exit 0 +fi + +cert_dir=/etc/kubernetes/certs + +echo -e "$CA_KEY" > ${cert_dir}/ca.key + +chown kube.kube ${cert_dir}/ca.key +chmod 400 ${cert_dir}/ca.key + diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh index a616f3b30c..1259ed3886 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh @@ -1,8 +1,18 @@ #!/bin/sh +. /etc/sysconfig/heat-params + # make sure we pick up any modified unit files systemctl daemon-reload +# if the certificate manager api is enabled, wait for the ca key to be handled +# by the heat container agent (required for the controller-manager) +while [ ! -f /etc/kubernetes/certs/ca.key ] && \ + [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" == "true" ]; do + echo "waiting for CA to be made available for certificate manager api" + sleep 2 +done + echo "starting services" for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler; do echo "activating service $service" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml index 3bf9600beb..355bf2a438 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -52,3 +52,5 @@ write_files: ETCD_LB_VIP="$ETCD_LB_VIP" DNS_SERVICE_IP="$DNS_SERVICE_IP" DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN" + CERT_MANAGER_API="$CERT_MANAGER_API" + CA_KEY="$CA_KEY" diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index a25f9db997..7faa6beb49 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -11,7 +11,10 @@ # under the License. from oslo_log import log as logging +from oslo_utils import strutils +from magnum.common.x509 import operations as x509 +from magnum.conductor.handlers.common import cert_manager from magnum.drivers.heat import k8s_template_def from magnum.drivers.heat import template_def from oslo_config import cfg @@ -88,6 +91,14 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): if label_value: extra_params[label] = label_value + cert_manager_api = cluster.labels.get('cert_manager_api') + if strutils.bool_from_string(cert_manager_api): + extra_params['cert_manager_api'] = cert_manager_api + ca_cert = cert_manager.get_cluster_ca_certificate(cluster) + extra_params['ca_key'] = x509.decrypt_key( + ca_cert.get_private_key(), + ca_cert.get_private_key_passphrase()).replace("\n", "\\n") + return super(K8sFedoraTemplateDefinition, self).get_params(context, cluster_template, cluster, extra_params=extra_params, diff --git a/magnum/drivers/heat/k8s_template_def.py b/magnum/drivers/heat/k8s_template_def.py index 00912efa2f..ed796bf3e0 100644 --- a/magnum/drivers/heat/k8s_template_def.py +++ b/magnum/drivers/heat/k8s_template_def.py @@ -113,7 +113,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition): 'prometheus_monitoring', 'grafana_admin_passwd', 'kube_dashboard_enabled', - 'etcd_volume_size'] + 'etcd_volume_size', + 'cert_manager_api'] for label in label_list: extra_params[label] = cluster.labels.get(label) diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index bbf22925f9..2609df789c 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -370,6 +370,17 @@ parameters: availability zone for master and nodes default: "" + cert_manager_api: + type: boolean + description: true if the kubernetes cert api manager should be enabled + default: false + + ca_key: + type: string + description: key of internal ca for the kube certificate api manager + default: "" + hidden: true + resources: ###################################################################### @@ -564,6 +575,8 @@ resources: openstack_ca: {get_param: openstack_ca} nodes_server_group_id: {get_resource: nodes_server_group} availability_zone: {get_param: availability_zone} + ca_key: {get_param: ca_key} + cert_manager_api: {get_param: cert_manager_api} ###################################################################### # diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 00c2ef68f8..fa6b82bdcf 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -273,6 +273,16 @@ parameters: availability zone for master and nodes default: "" + ca_key: + type: string + description: key of internal ca for the kube certificate api manager + hidden: true + + cert_manager_api: + type: boolean + description: true if the kubernetes cert api manager should be enabled + default: false + resources: master_wait_handle: @@ -359,6 +369,8 @@ resources: "$ETCD_LB_VIP": {get_param: etcd_lb_vip} "$DNS_SERVICE_IP": {get_param: dns_service_ip} "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain} + "$CERT_MANAGER_API": {get_param: cert_manager_api} + "$CA_KEY": {get_param: ca_key} install_openstack_ca: type: OS::Heat::SoftwareConfig @@ -483,6 +495,7 @@ resources: - config: {get_resource: configure_docker_storage} - config: {get_resource: configure_kubernetes} - config: {get_resource: add_proxy} + - config: {get_resource: start_container_agent} - config: {get_resource: enable_services} - config: {get_resource: write_network_config} - config: {get_resource: network_config_service} @@ -490,7 +503,6 @@ resources: - config: {get_resource: kube_apiserver_to_kubelet_role} - config: {get_resource: core_dns_service} - config: {get_resource: kube_ui_service} - - config: {get_resource: start_container_agent} - config: {get_resource: master_wc_notify} enable_prometheus_monitoring: @@ -511,6 +523,24 @@ resources: server: {get_resource: kube-master} actions: ['CREATE'] + enable_cert_manager_api: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + str_replace: + template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager} + params: + "$CA_KEY": {get_param: ca_key} + + enable_cert_manager_api_deployment: + type: OS::Heat::SoftwareDeployment + properties: + signal_transport: HEAT_SIGNAL + config: {get_resource: enable_cert_manager_api} + server: {get_resource: kube-master} + actions: ['CREATE'] + ###################################################################### # # a single kubernetes master. diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index e88b6dbe0a..10ff10d6f6 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -97,7 +97,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'grafana_admin_passwd': 'fake_pwd', 'kube_dashboard_enabled': 'True', 'docker_volume_type': 'lvmdriver-1', - 'availability_zone': 'az_1'}, + 'availability_zone': 'az_1', + 'cert_manager_api': 'False'}, 'master_flavor_id': 'master_flavor_id', 'flavor_id': 'flavor_id', } @@ -177,7 +178,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'kube_dashboard_enabled': 'True', 'docker_volume_type': 'lvmdriver-1', 'etcd_volume_size': None, - 'availability_zone': 'az_1'}, + 'availability_zone': 'az_1', + 'cert_manager_api': 'False'}, 'http_proxy': 'http_proxy', 'https_proxy': 'https_proxy', 'no_proxy': 'no_proxy', @@ -233,7 +235,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'verify_ca': True, 'openstack_ca': '', "nodes_affinity_policy": "soft-anti-affinity", - 'availability_zone': 'az_1' + 'availability_zone': 'az_1', + 'cert_manager_api': 'False', } if missing_attr is not None: expected.pop(mapping[missing_attr], None) @@ -331,7 +334,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'verify_ca': True, 'openstack_ca': '', "nodes_affinity_policy": "soft-anti-affinity", - 'availability_zone': 'az_1' + 'availability_zone': 'az_1', + 'cert_manager_api': 'False', } self.assertEqual(expected, definition) @@ -416,7 +420,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'verify_ca': True, 'openstack_ca': '', "nodes_affinity_policy": "soft-anti-affinity", - 'availability_zone': 'az_1' + 'availability_zone': 'az_1', + 'cert_manager_api': 'False', } self.assertEqual(expected, definition) self.assertEqual( @@ -494,7 +499,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'insecure_registry_url': '10.0.0.1:5000', 'kube_version': 'fake-version', 'verify_ca': True, - 'openstack_ca': '' + 'openstack_ca': '', + 'cert_manager_api': 'False', } self.assertEqual(expected, definition) self.assertEqual( @@ -567,7 +573,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'insecure_registry_url': '10.0.0.1:5000', 'kube_version': 'fake-version', 'verify_ca': True, - 'openstack_ca': '' + 'openstack_ca': '', + 'cert_manager_api': 'False', } self.assertEqual(expected, definition) self.assertEqual( @@ -741,7 +748,8 @@ class TestClusterConductorWithK8s(base.TestCase): 'verify_ca': True, 'openstack_ca': '', "nodes_affinity_policy": "soft-anti-affinity", - 'availability_zone': 'az_1' + 'availability_zone': 'az_1', + 'cert_manager_api': 'False', } self.assertEqual(expected, definition) self.assertEqual( diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 726960b822..66558d5bd1 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -274,6 +274,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'container_infra_prefix') availability_zone = mock_cluster.labels.get( 'availability_zone') + cert_manager_api = mock_cluster.labels.get('cert_manager_api') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -301,7 +302,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'container_infra_prefix': container_infra_prefix, 'nodes_affinity_policy': 'soft-anti-affinity', 'availability_zone': availability_zone, - }} + 'cert_manager_api': cert_manager_api}} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster, @@ -362,6 +363,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'container_infra_prefix') availability_zone = mock_cluster.labels.get( 'availability_zone') + cert_manager_api = mock_cluster.labels.get('cert_manager_api') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -391,7 +393,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase): 'container_infra_prefix': container_infra_prefix, 'nodes_affinity_policy': 'soft-anti-affinity', 'availability_zone': availability_zone, - }} + 'cert_manager_api': cert_manager_api}} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, mock_cluster, diff --git a/releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml b/releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml new file mode 100644 index 0000000000..0822688c88 --- /dev/null +++ b/releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add new label 'cert_manager_api' enabling the kubernetes certificate + manager api.