[k8s] allow enabling kubernetes cert manager api

Add a new label 'cert_manager_api' to kubernetes clusters controlling the
enable/disable of the kubernetes certificate manager api.

The same cluster cert/key pair is used by this api. The heat agent is used
to install the key in the master node(s), as this is required for kubernetes
to later sign new certificate requests.

The master template init order is changed so the heat agent is launched
previous to enabling the services - the controller manager requires the CA key
to be locally available before being launched.

Change-Id: Ibf85147316e3a194d8a3f92cbb4ae9ce8e16c98f
Partial-Bug: #1734318
changes/18/529818/8
Ricardo Rocha 5 years ago committed by Spyros Trigazis (strigazi)
parent c6e7b290ab
commit faa9e90402
  1. 7
      doc/source/user/index.rst
  2. 4
      magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
  3. 15
      magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager
  4. 10
      magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
  5. 2
      magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
  6. 11
      magnum/drivers/heat/k8s_fedora_template_def.py
  7. 3
      magnum/drivers/heat/k8s_template_def.py
  8. 13
      magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
  9. 32
      magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
  10. 24
      magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
  11. 6
      magnum/tests/unit/drivers/test_template_definition.py
  12. 5
      releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml

@ -348,10 +348,11 @@ the table are linked to more details elsewhere in the user guide.
+---------------------------------------+--------------------+---------------+
| `container_infra_prefix`_ | see below | "" |
+---------------------------------------+--------------------+---------------+
+---------------------------------------+--------------------+---------------+
| `availability_zone`_ | AZ for the cluster | "" |
| | nodes | |
+---------------------------------------+--------------------+---------------+
| `cert_manager_api`_ | see below | false |
+---------------------------------------+--------------------+---------------+
Cluster
-------
@ -1113,6 +1114,10 @@ _`kube_dashboard_enabled`
This label triggers the deployment of the kubernetes dashboard.
The default value is 1, meaning it will be enabled.
_`cert_manager_api`
This label enables the kubernetes `certificate manager api
<https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/>`_.
External load balancer for services
-----------------------------------

@ -57,6 +57,10 @@ if [ -n "$TRUST_ID" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/kubernetes/kube_openstack_config --cloud-provider=openstack"
fi
if [ -n "$CERT_MANAGER_API" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-signing-cert-file=$CERT_DIR/ca.crt --cluster-signing-key-file=$CERT_DIR/ca.key"
fi
sed -i '
/^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/
/^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#

@ -0,0 +1,15 @@
#!/bin/bash
. /etc/sysconfig/heat-params
if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" = "false" ]; then
exit 0
fi
cert_dir=/etc/kubernetes/certs
echo -e "$CA_KEY" > ${cert_dir}/ca.key
chown kube.kube ${cert_dir}/ca.key
chmod 400 ${cert_dir}/ca.key

@ -1,8 +1,18 @@
#!/bin/sh
. /etc/sysconfig/heat-params
# make sure we pick up any modified unit files
systemctl daemon-reload
# if the certificate manager api is enabled, wait for the ca key to be handled
# by the heat container agent (required for the controller-manager)
while [ ! -f /etc/kubernetes/certs/ca.key ] && \
[ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" == "true" ]; do
echo "waiting for CA to be made available for certificate manager api"
sleep 2
done
echo "starting services"
for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler; do
echo "activating service $service"

@ -52,3 +52,5 @@ write_files:
ETCD_LB_VIP="$ETCD_LB_VIP"
DNS_SERVICE_IP="$DNS_SERVICE_IP"
DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
CERT_MANAGER_API="$CERT_MANAGER_API"
CA_KEY="$CA_KEY"

@ -11,7 +11,10 @@
# under the License.
from oslo_log import log as logging
from oslo_utils import strutils
from magnum.common.x509 import operations as x509
from magnum.conductor.handlers.common import cert_manager
from magnum.drivers.heat import k8s_template_def
from magnum.drivers.heat import template_def
from oslo_config import cfg
@ -88,6 +91,14 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
if label_value:
extra_params[label] = label_value
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
return super(K8sFedoraTemplateDefinition,
self).get_params(context, cluster_template, cluster,
extra_params=extra_params,

@ -113,7 +113,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
'prometheus_monitoring',
'grafana_admin_passwd',
'kube_dashboard_enabled',
'etcd_volume_size']
'etcd_volume_size',
'cert_manager_api']
for label in label_list:
extra_params[label] = cluster.labels.get(label)

@ -370,6 +370,17 @@ parameters:
availability zone for master and nodes
default: ""
cert_manager_api:
type: boolean
description: true if the kubernetes cert api manager should be enabled
default: false
ca_key:
type: string
description: key of internal ca for the kube certificate api manager
default: ""
hidden: true
resources:
######################################################################
@ -564,6 +575,8 @@ resources:
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
availability_zone: {get_param: availability_zone}
ca_key: {get_param: ca_key}
cert_manager_api: {get_param: cert_manager_api}
######################################################################
#

@ -273,6 +273,16 @@ parameters:
availability zone for master and nodes
default: ""
ca_key:
type: string
description: key of internal ca for the kube certificate api manager
hidden: true
cert_manager_api:
type: boolean
description: true if the kubernetes cert api manager should be enabled
default: false
resources:
master_wait_handle:
@ -359,6 +369,8 @@ resources:
"$ETCD_LB_VIP": {get_param: etcd_lb_vip}
"$DNS_SERVICE_IP": {get_param: dns_service_ip}
"$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
"$CERT_MANAGER_API": {get_param: cert_manager_api}
"$CA_KEY": {get_param: ca_key}
install_openstack_ca:
type: OS::Heat::SoftwareConfig
@ -483,6 +495,7 @@ resources:
- config: {get_resource: configure_docker_storage}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: add_proxy}
- config: {get_resource: start_container_agent}
- config: {get_resource: enable_services}
- config: {get_resource: write_network_config}
- config: {get_resource: network_config_service}
@ -490,7 +503,6 @@ resources:
- config: {get_resource: kube_apiserver_to_kubelet_role}
- config: {get_resource: core_dns_service}
- config: {get_resource: kube_ui_service}
- config: {get_resource: start_container_agent}
- config: {get_resource: master_wc_notify}
enable_prometheus_monitoring:
@ -511,6 +523,24 @@ resources:
server: {get_resource: kube-master}
actions: ['CREATE']
enable_cert_manager_api:
type: OS::Heat::SoftwareConfig
properties:
group: script
config:
str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager}
params:
"$CA_KEY": {get_param: ca_key}
enable_cert_manager_api_deployment:
type: OS::Heat::SoftwareDeployment
properties:
signal_transport: HEAT_SIGNAL
config: {get_resource: enable_cert_manager_api}
server: {get_resource: kube-master}
actions: ['CREATE']
######################################################################
#
# a single kubernetes master.

@ -97,7 +97,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'grafana_admin_passwd': 'fake_pwd',
'kube_dashboard_enabled': 'True',
'docker_volume_type': 'lvmdriver-1',
'availability_zone': 'az_1'},
'availability_zone': 'az_1',
'cert_manager_api': 'False'},
'master_flavor_id': 'master_flavor_id',
'flavor_id': 'flavor_id',
}
@ -177,7 +178,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_dashboard_enabled': 'True',
'docker_volume_type': 'lvmdriver-1',
'etcd_volume_size': None,
'availability_zone': 'az_1'},
'availability_zone': 'az_1',
'cert_manager_api': 'False'},
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
@ -233,7 +235,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -331,7 +334,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
@ -416,7 +420,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -494,7 +499,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'insecure_registry_url': '10.0.0.1:5000',
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': ''
'openstack_ca': '',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -567,7 +573,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'insecure_registry_url': '10.0.0.1:5000',
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': ''
'openstack_ca': '',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -741,7 +748,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(

@ -274,6 +274,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix')
availability_zone = mock_cluster.labels.get(
'availability_zone')
cert_manager_api = mock_cluster.labels.get('cert_manager_api')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -301,7 +302,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity',
'availability_zone': availability_zone,
}}
'cert_manager_api': cert_manager_api}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
@ -362,6 +363,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix')
availability_zone = mock_cluster.labels.get(
'availability_zone')
cert_manager_api = mock_cluster.labels.get('cert_manager_api')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -391,7 +393,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity',
'availability_zone': availability_zone,
}}
'cert_manager_api': cert_manager_api}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,

@ -0,0 +1,5 @@
---
features:
- |
Add new label 'cert_manager_api' enabling the kubernetes certificate
manager api.
Loading…
Cancel
Save